Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): serve GGUF checkpoints on vendored ggml-cuda kernels (#690)

Xiaoze Fane5d3bd3fafb7
diff --git a/benchmarks/bench_load_weight_generic.py b/benchmarks/bench_load_weight_generic.py
index 4e4c5fe0..89ddb307 100644
--- a/benchmarks/bench_load_weight_generic.py
+++ b/benchmarks/bench_load_weight_generic.py
@@ -179,7 +179,7 @@ def _bench_load(mode: str, model_path: str, *, parallel: bool, workers: int, chu
t = time.perf_counter()
try:
banks = load_expert_banks(model_path, mc, device=torch.device("cuda", torch.cuda.current_device()),
- dtype=torch.bfloat16, parallel=parallel,
+ parallel=parallel,
workers=workers, chunk=chunk)
except NotImplementedError as e:
s.stop()
diff --git a/docs/cli.md b/docs/cli.md
index 44b0e30a..4918a317 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -189,7 +189,7 @@ ft checkpoint --model <hf_dir> --out <ftw_dir> [--dtype bfloat16] [--moe-backend
Converts an HF safetensors checkpoint to FTW, FreeToken's self-contained
fast-load format; point `ft serve --model` at the output dir. MoE experts are
always packed into expert banks, which serve the fused and the offload
-strategies alike (GGUF and DeepSeek-V4 experts serve offload only);
+strategies alike (DeepSeek-V4 experts serve offload only);
`--moe-backend` is kept for compatibility. See the FTW caveats in
[models.md](models.md#notes); FTW files from older builds can be repaired with
[scripts/ftw_hotfix.py](ftw-hotfix.md) instead of reconverting.
diff --git a/docs/models.md b/docs/models.md
index cdadc2dc..d3f26a08 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -42,7 +42,7 @@ These families accept image input by default; pass `--text-model-only` to skip t
`ft serve --moe-strategy {auto,fused,offload,cpu,hybrid}` (`--moe-backend` is the deprecated old spelling):
- **fused** — experts resident on GPU (needs the VRAM). Serves every expert
- format except GGUF and DeepSeek-V4's.
+ format except DeepSeek-V4's.
- **offload** — experts live in host RAM, an LRU cache of expert slots on GPU;
misses stream over PCIe.
- **cpu** — misses are computed on the CPU instead of fetched.
@@ -56,7 +56,7 @@ These families accept image input by default; pass `--text-model-only` to skip t
The GPU and the CPU share one DRAM, so offload only copies experts between two
names for the same memory. On these GPUs `auto` resolves every MoE model to
-`fused`: the default is the resident path only. GGUF and DeepSeek-V4 experts
+`fused`: the default is the resident path only. DeepSeek-V4 experts
have no resident path and stay on `offload`. Offload-only flags
(`--moe-cpu-layers`, `--moe-cache-size` / `-rate` / `-auto`,
`--moe-prefill-hit-d2d`, `--disable-moe-prefill-overlap`) are ignored with a
diff --git a/pyproject.toml b/pyproject.toml
index b308045f..ab1dfd07 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,7 +40,6 @@ dependencies = [
"fastapi>=0.115,<1",
# slot_cache: the device-side LRU admission kernel behind the MoE expert cache.
"flashlib==0.4.0", # pin it
- "gguf>=0.19,<1",
"huggingface_hub>=1.5,<2",
"msgpack>=1.1,<2",
"modelscope>=1.37,<2",
@@ -113,7 +112,7 @@ version = {attr = "freetoken.version.__version__"}
where = ["python"]
[tool.setuptools.package-data]
-"*" = ["csrc/**/*", "moe/configs/**/*.json"]
+"*" = ["csrc/**/*", "moe/configs/**/*.json", "third_party/*/LICENSE", "third_party/*/VERSION", "third_party/*/freetoken.patch"]
[tool.pytest.ini_options]
testpaths = ["tests"]
diff --git a/python/freetoken/checkpoint/convert.py b/python/freetoken/checkpoint/convert.py
index 152f3871..054fade4 100644
--- a/python/freetoken/checkpoint/convert.py
+++ b/python/freetoken/checkpoint/convert.py
@@ -247,7 +247,7 @@ def convert_checkpoint(
names = legacy_bank_names(legacy_format_for(method.kind, method.kernel.name)) if method is not None else {}
sink = _ConvertSink(writer, names=names)
banks = load_expert_banks(
- model_path, mc, method=method, device=dev, dtype=dtype, layer_sink=sink
+ model_path, mc, method=method, device=dev, layer_sink=sink
)
quant_format = banks.quant_format
if banks.streamed:
diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.py
index ae2f1fd6..a6c47dd0 100644
--- a/python/freetoken/engine/config.py
+++ b/python/freetoken/engine/config.py
@@ -141,7 +141,7 @@ class EngineConfig:
hf_config = copy.copy(self.hf_config)
built = {e.config_key for e in self.active_encoders}
for key in set(ENCODER_SECTIONS) | {e.config_key for e in self.model_spec.encoders}:
- if key not in built:
+ if key not in built and getattr(hf_config, key, None) is not None:
setattr(hf_config, key, None)
spec = self.model_spec
quant = checkpoint_quant_config(self.model_path, hf_config, spec)
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index 2757b10d..81f42dba 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -15,7 +15,7 @@ from freetoken.distributed import destroy_distributed, enable_pynccl_distributed
from freetoken.gpu_select import gpu_identity
from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint
from freetoken.layers import OffloadMoELayer, iter_moe_layers, set_rope_device
-from freetoken.layers.quantization import LayerKind, QuantBackend, finalize_quant, set_quant_backend
+from freetoken.layers.quantization import LayerKind, QuantBackend, QuantKind, finalize_quant, set_quant_backend
from freetoken.moe.offload_cache import iter_offload_moe_layers
from freetoken.mm.config import ENCODER_SECTIONS
from freetoken.models import create_model, load_weight
@@ -304,7 +304,7 @@ def _make_dummy_weight_state_dict(
# MXFP8 e8m0 exponent codes: 127 encodes scale 1.0; zeros would collapse
# every scale to 2^-127 and zero the model. Scoped BY NAME: other uint8
# buffers are packed payloads whose bytes mean something else entirely
- # (GGUF qweight blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they
+ # (GGUF blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they
# keep the benign all-zeros fill below.
state_dict[key] = torch.full(param.shape, 127, dtype=param.dtype, device=device)
else:
@@ -603,7 +603,6 @@ class Engine:
config.model_config,
method=layers[0].quant_method,
device=self.device,
- dtype=self.dtype,
dummy=config.use_dummy_weight,
parallel={"serial": False, "parallel": True}.get(config.expert_load, None),
resident=True,
@@ -735,10 +734,8 @@ class Engine:
config.model_config,
method=method,
device=self.device,
- dtype=self.dtype,
dummy=config.use_dummy_weight,
parallel=expert_parallel,
- decode_target=("cpu" if decode_target in ("cpu", "hybrid") else "gpu"),
layer_residency=requested_residency,
)
except PinFailed as exc:
@@ -808,7 +805,8 @@ class Engine:
f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "
f"binds {method.kind} / {method.kernel.name}; reconvert or select that kernel"
)
- layout = method.layout()
+ # GGUF layers keep their own ggml types, so the cache reads each layer's layout off its banks
+ layout = None if method.kind is QuantKind.GGUF else method.layout()
max_slots = method.slot_limit()
cache = OffloadMoeCache(
# Models with leading dense layers (GLM-4) only have experts on the MoE
@@ -830,7 +828,7 @@ class Engine:
)
# before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set
cache.cpu_layer_ids = cpu_layer_ids
- cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=banks.layer_tags)
+ cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=banks.layer_tags, row_tail_bytes=banks.row_tail_bytes)
cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
if decode_target == "hybrid":
self._resolve_hybrid_fetch(config, cache)
@@ -839,7 +837,6 @@ class Engine:
cache.collect_stats = config.moe_collect_stats
layers = attach_offload_moe_cache(self.model, cache)
assert len(layers) == config.model_config.num_moe_layers
- cache.zero_inactive_route_slots = cache.decode_target == "hybrid" and not all(layer.skips_inactive_slots for layer in layers)
if cache.decode_target in ("cpu", "hybrid"):
self._init_cpu_moe_executor(config, cache, layers)
self.ctx.moe_offload_cache = cache
@@ -907,6 +904,9 @@ class Engine:
# round a batch up to the largest captured size; cover both.
max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)
method = sample.quant_method
+ formats = {layer.quant_method.cpu_format for layer in layers}
+ if len(formats) > 1:
+ raise NotImplementedError(f"CPU MoE backend needs one expert format across the MoE layers, got {sorted(map(str, formats))}")
executor = CpuMoeExecutor(
cache,
top_k=sample.top_k,
@@ -917,9 +917,8 @@ class Engine:
device=self.device,
swiglu_alpha=float(sample.alpha),
swiglu_limit=sample.limit,
- # FIXME: the None branch serves GGUF q4_0 banks, which have no quant method yet; drop it once GGUF joins the quant path
- fmt=method.cpu_format if method is not None else None,
- act_block=method.cfg.act_block if method is not None else None,
+ fmt=method.cpu_format,
+ act_block=method.cfg.act_block,
)
cache.set_cpu_executor(executor)
self.cpu_moe_executor = executor
@@ -1255,11 +1254,11 @@ def _is_unified_memory_gpu(index: "int | None" = None) -> bool:
return False
-_RESIDENT_EXPERT_QUANTS = ("none", "fp8_block", "nvfp4")
+_RESIDENT_EXPERT_QUANTS = ("none", "fp8_block", "nvfp4", "gguf")
def _fused_resident_ok(model_config) -> bool:
- """Whether the resident ('fused') MoE path can hold this model's experts; GGUF q4_0 and DeepSeek-V4 experts stay offload."""
+ """Whether the resident ('fused') MoE path can hold this model's experts; DeepSeek-V4 experts stay offload."""
if getattr(model_config, "expert_quant", "none") not in _RESIDENT_EXPERT_QUANTS:
return False
return getattr(model_config, "moe_weight_format", None) in (None, "bf16", "mxfp4")
@@ -1752,7 +1751,7 @@ def _adjust_config(config: EngineConfig):
if is_moe and config.moe_strategy == "auto":
# A MoE model always defaults to the offload family: experts stream from pinned host
# banks into an auto-sized GPU slot cache, which is the only default that serves a model
- # bigger than the GPU. The resident 'fused' path (every expert format but GGUF q4_0 and
+ # bigger than the GPU. The resident 'fused' path (every expert format but
# DeepSeek-V4's) is still reachable, but only when asked for explicitly
# -- auto never picks it, because nothing here knows whether the experts would fit in
# HBM and a wrong guess is a weight-load OOM rather than a slower-but-working run.
@@ -1790,7 +1789,7 @@ def _adjust_config(config: EngineConfig):
# offload -- same auto-sized GPU slot cache (_resolve_auto_moe_cache_size), plus a
# host-RAM CPU executor -- so this never raises the OOM risk; with no profile (or one
# from different hardware) it stays offload. offload remains the always-safe fallback.
- # Key the lookup on the real expert format: mxfp4/q4_0 live in moe_weight_format when
+ # Key the lookup on the real expert format: mxfp4 lives in moe_weight_format when
# expert_quant is "none", and "none" with no weight format means plain bf16 experts.
moe_wfmt = getattr(model_config, "moe_weight_format", None)
bench_fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16")
diff --git a/python/freetoken/kernel/aot.py b/python/freetoken/kernel/aot.py
index bbc2d2c6..76250af7 100644
--- a/python/freetoken/kernel/aot.py
+++ b/python/freetoken/kernel/aot.py
@@ -143,6 +143,15 @@ def _radix_spec() -> KernelSpec:
return KernelSpec(name=_make_name("radix"), build=build)
+def _ggml_spec() -> KernelSpec:
+ def build(build_directory: pathlib.Path) -> object:
+ from .ggml import load_module
+
+ return load_module(str(build_directory))
+
+ return KernelSpec(name=_make_name("ggml"), build=build)
+
+
def default_kernel_specs() -> tuple[KernelSpec, ...]:
specs: list[KernelSpec] = []
specs.extend(_store_spec(element_size) for element_size in DEFAULT_STORE_ELEMENT_SIZES)
@@ -158,6 +167,7 @@ def default_kernel_specs() -> tuple[KernelSpec, ...]:
specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=64))
specs.append(_batch_memcpy_spec())
specs.append(_radix_spec())
+ specs.append(_ggml_spec())
names = [spec.name for spec in specs]
duplicates = {name for name in names if names.count(name) > 1}
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index ecd1f850..308c2d68 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -16,7 +16,7 @@ JIT, which needs nvcc):
DSV4 writes its MLA latent via torch scatter and contributes nothing.
- index: ``element_size = hidden_size * 2`` (bf16 embedding row) paired with
the runtime ``num_splits_for`` rule (layers/embedding.py -> kernel/index.py).
- DSV4 (plain nn.Embedding) and GGUF embeddings (GGUFEmbedding) bypass it.
+ DSV4 (plain nn.Embedding) and GGUF embeddings (ggml get_rows) bypass it.
The whole table targets the shipped serving configuration: TP=1 (TP>1 shards
kv heads, shrinking the store row) and a 2-byte compute dtype (``--dtype
@@ -91,9 +91,6 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
"down": H * I,
"down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2,
}
- if fmt == "q4_0":
- # gemma4/gguf.py _q4_0_expert_specs: GGML Q4_0 rows, 32 elems -> 18 bytes
- return {"gate_up": 2 * I * (H // 32 * 18), "down": H * (I // 32 * 18)}
if fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"):
# models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp16
# per-row globals; marlin/b12x repacks are byte-identical with the globals
@@ -205,6 +202,8 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
top_k=10,
moe_intermediate_size=640,
expert_formats=(*_NVFP4_FORMATS, "fp8_block"),
+ # GGUF expert rows differ per layer and reach the fused copy as runtime sizes
+ arch_aliases=("Qwen4ExpGGUFForCausalLM",),
),
AotModel(
name="google/gemma-4-26B-A4B-it",
@@ -215,9 +214,8 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
kv_groups=((2, 512), (8, 256)),
top_k=8,
moe_intermediate_size=704,
- # q4_0 = the Q4_0 GGUF release (Gemma4GGUFForCausalLM); its GGUFEmbedding
- # path skips indexing(), but the bf16 safetensors path needs the variant.
- expert_formats=("bf16", "q4_0"),
+ expert_formats=("bf16",),
+ # GGUF expert rows differ per layer and reach the fused copy as runtime sizes
arch_aliases=("Gemma4ForCausalLM", "Gemma4GGUFForCausalLM"),
),
AotModel(
diff --git a/python/freetoken/kernel/csrc/ggml/LICENSE b/python/freetoken/kernel/csrc/ggml/LICENSE
new file mode 100644
index 00000000..e7dca554
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023-2026 The ggml authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/python/freetoken/kernel/csrc/ggml/VERSION b/python/freetoken/kernel/csrc/ggml/VERSION
new file mode 100644
index 00000000..6eabb0c1
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/VERSION
@@ -0,0 +1,13 @@
+llama.cpp 8c2134a8755d8a9792af513dab20282affe903e4 (2026-10-14)
+
+Only the ggml files that the GGUF ops (freetoken/ggml_ops.cu, built by kernel/ggml.py) compile or include:
+ggml/include -> include, ggml/src -> src, ggml/src/ggml-cuda -> ggml-cuda.
+
+Local changes, kept in freetoken.patch (already applied):
+- quantize.cu/.cuh: the q8_1 quantize kernels read fp32, bf16 or fp16 activations.
+- mmq.cu/.cuh: MMQ takes bf16 or fp16 activations, except on the native fp4 path. Its kernels are built for fp32 and bf16 results (dst_t); a bf16 result that needs the stream-k fixup is computed in fp32 first, and so is an fp16 result unless GGML_MMQ_F16_DST builds fp16 kernels too.
+- mmvq.cu/.cuh: MMVQ takes bf16 or fp16 activations, can write bf16 or fp16 results, and can reuse activations already quantized to q8_1 (src1_q8_1_pre).
+- mmq.cu/.cuh, mmid.cu/.cuh: MoE MMQ runs a flat grid over only the (expert, column tile) pairs that hold tokens, listed by mm_ids_tile_map; ncols_max > 0 keeps the regular grid.
+- common.cuh: the native fp4 MMA (BLACKWELL_MMA_AVAILABLE, blackwell_mma_available) is off unless GGML_CUDA_FP4_MMA is defined; it needs an sm_120a/sm_120f build.
+
+To update: copy the same files from a newer llama.cpp, apply freetoken.patch, and update this file.
diff --git a/python/freetoken/kernel/csrc/ggml/freetoken.patch b/python/freetoken/kernel/csrc/ggml/freetoken.patch
new file mode 100644
index 00000000..3dc715fe
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/freetoken.patch
@@ -0,0 +1,994 @@
+--- a/ggml/src/ggml-cuda/common.cuh
++++ b/ggml/src/ggml-cuda/common.cuh
+@@ -293,7 +293,8 @@
+ #define AMPERE_MMA_AVAILABLE
+ #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+
+-#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN
++// FreeToken: the native fp4 MMA needs an sm_120a/sm_120f build, so it is off unless GGML_CUDA_FP4_MMA is defined; FP4 weights then take Q8_1 activations
++#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN && defined(GGML_CUDA_FP4_MMA)
+ # define BLACKWELL_MMA_AVAILABLE
+ #endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL
+
+@@ -374,8 +375,13 @@
+ }
+
+ static bool blackwell_mma_available(const int cc) {
++#ifdef GGML_CUDA_FP4_MMA
+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL &&
+ ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;
++#else
++ GGML_UNUSED(cc);
++ return false;
++#endif // GGML_CUDA_FP4_MMA
+ }
+
+ // Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes.
+--- a/ggml/src/ggml-cuda/mmid.cu
++++ b/ggml/src/ggml-cuda/mmid.cu
+@@ -177,3 +177,45 @@
+ break;
+ }
+ }
++
++// Lists the (expert, column tile) pairs that hold columns for a flat MoE grid, in expert order.
++// The other ntiles grid columns get tile jt_none of expert 0, which starts past its columns, so mul_mat_q skips them.
++template <int nthreads>
++__launch_bounds__(nthreads, 1)
++static __global__ void mm_ids_tile_map(
++ const int32_t * __restrict__ expert_bounds, int2 * __restrict__ tile_map, const int n_experts, const int J, const int ntiles, const int jt_none) {
++ __shared__ int scan[nthreads];
++
++ int start = 0; // First tile of the current chunk of experts.
++ for (int e0 = 0; e0 < n_experts; e0 += nthreads) {
++ const int e = e0 + threadIdx.x;
++ const int n = e < n_experts ? (expert_bounds[e + 1] - expert_bounds[e] + J - 1) / J : 0;
++
++ scan[threadIdx.x] = n;
++ __syncthreads();
++#pragma unroll
++ for (int offset = 1; offset < nthreads; offset *= 2) {
++ const int add = threadIdx.x >= offset ? scan[threadIdx.x - offset] : 0;
++ __syncthreads();
++ scan[threadIdx.x] += add;
++ __syncthreads();
++ }
++
++ const int first = start + scan[threadIdx.x] - n;
++ for (int jt = 0; jt < n; ++jt) {
++ tile_map[first + jt] = make_int2(e, jt);
++ }
++ start += scan[nthreads - 1];
++ __syncthreads();
++ }
++
++ for (int t = start + threadIdx.x; t < ntiles; t += nthreads) {
++ tile_map[t] = make_int2(0, jt_none);
++ }
++}
++
++void ggml_cuda_launch_mm_ids_tile_map(
++ const int32_t * expert_bounds, int2 * tile_map, const int n_experts, const int J, const int ntiles, const int jt_none, cudaStream_t stream) {
++ constexpr int nthreads = 512;
++ mm_ids_tile_map<nthreads><<<1, nthreads, 0, stream>>>(expert_bounds, tile_map, n_experts, J, ntiles, jt_none);
++}
+--- a/ggml/src/ggml-cuda/mmid.cuh
++++ b/ggml/src/ggml-cuda/mmid.cuh
+@@ -3,3 +3,6 @@
+ void ggml_cuda_launch_mm_ids_helper(
+ const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds,
+ int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream);
++
++void ggml_cuda_launch_mm_ids_tile_map(
++ const int32_t * expert_bounds, int2 * tile_map, int n_experts, int J, int ntiles, int jt_none, cudaStream_t stream);
+--- a/ggml/src/ggml-cuda/mmq.cu
++++ b/ggml/src/ggml-cuda/mmq.cu
+@@ -132,9 +132,11 @@
+ }
+
+ void ggml_cuda_mul_mat_q(
+- ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
+- GGML_ASSERT( src1->type == GGML_TYPE_F32);
+- GGML_ASSERT( dst->type == GGML_TYPE_F32);
++ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
++ int64_t ncols_max) {
++ // FreeToken: bf16 and fp16 activations are quantized directly, except for the native fp4 paths; dst is fp32, bf16 or fp16
++ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);
++ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);
+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+@@ -156,7 +158,9 @@
+
+ const char * src0_d = (const char *) src0->data;
+ const float * src1_d = (const float *) src1->data;
+- float * dst_d = (float *) dst->data;
++ const mmq_dst dst_d = {dst->data, dst->type};
++ const nv_bfloat16 * src1_bf16 = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1->data : nullptr;
++ const half * src1_f16 = src1->type == GGML_TYPE_F16 ? (const half *) src1->data : nullptr;
+
+ // If src0 is a temporary compute buffer, clear any potential padding.
+ if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
+@@ -183,6 +187,7 @@
+ const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc);
+
+ const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;
++ GGML_ASSERT(!use_native_fp4 || src1->type == GGML_TYPE_F32);
+ const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);
+ const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ;
+
+@@ -192,7 +197,8 @@
+ int64_t ncols_opt = ne11;
+ if (ids) {
+ const int64_t n_expert_used = ids->ne[0];
+- ncols_opt = ne12;
++ // a flat grid runs only the tiles that hold tokens, so it sizes them for the average expert
++ ncols_opt = ncols_max > 0 ? ncols_max : (ne12*n_expert_used + ne02 - 1) / ne02;
+
+ // Each expert only sees ne12*n_expert_used/ne02 tokens on average.
+ // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.
+@@ -249,6 +255,12 @@
+ quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded,
+ ne11, ne12, ne13, stream);
+
++ } else if (src1_bf16) {
++ quantize_mmq_q8_1_cuda(src1_bf16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
++ ne11, ne12, ne13, stream);
++ } else if (src1_f16) {
++ quantize_mmq_q8_1_cuda(src1_f16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
++ ne11, ne12, ne13, stream);
+ } else {
+ quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
+ ne11, ne12, ne13, stream);
+@@ -299,6 +311,17 @@
+ CUDA_CHECK(cudaGetLastError());
+ }
+
++ // Every expert's column tiles fit in ntx_flat grid columns, so a flat grid needs no host sync to size it.
++ ggml_cuda_pool_alloc<int2> tile_map(ctx.pool());
++ int ntx_flat = 0;
++ if (ncols_max <= 0) {
++ const int jt_none = (ne_get_rows + J_best - 1) / J_best;
++ ntx_flat = jt_none + std::min<int64_t>(ne02, ne_get_rows);
++ tile_map.alloc(ntx_flat);
++ ggml_cuda_launch_mm_ids_tile_map(expert_bounds.get(), tile_map.get(), ne02, J_best, ntx_flat, jt_none, stream);
++ CUDA_CHECK(cudaGetLastError());
++ }
++
+ const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;
+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);
+ ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());
+@@ -325,9 +348,21 @@
+ quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13,
+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
+ }
++ } else if (dedup_bcast && src1_bf16) {
++ quantize_scatter_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
++ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
++ } else if (dedup_bcast && src1_f16) {
++ quantize_scatter_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
++ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
+ } else if (dedup_bcast) {
+ quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
++ } else if (src1_bf16) {
++ quantize_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
++ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
++ } else if (src1_f16) {
++ quantize_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
++ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
+ } else {
+ quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
+@@ -347,7 +382,7 @@
+ ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,
+ ne02, ne02, s02, s12, s2,
+ ne03, ne13, s03, s13, s3,
+- ne12, J_best};
++ ncols_max > 0 ? ncols_max : ne12, J_best, tile_map.ptr, ntx_flat};
+
+ ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);
+ }
+--- a/ggml/src/ggml-cuda/mmq.cuh
++++ b/ggml/src/ggml-cuda/mmq.cuh
+@@ -10,6 +10,17 @@
+ #define MMQ_ITER_K_FP4 512
+ #define MMQ_NWARPS 8
+
++// FreeToken: MMQ writes fp32, bf16 or fp16 results (dst_t); the sums stay fp32 until the store
++template <typename T> static __device__ __forceinline__ T mmq_cast(float v);
++template <> __device__ __forceinline__ float mmq_cast<float>(float v) { return v; }
++template <> __device__ __forceinline__ nv_bfloat16 mmq_cast<nv_bfloat16>(float v) { return __float2bfloat16(v); }
++template <> __device__ __forceinline__ half mmq_cast<half>(float v) { return __float2half(v); }
++
++struct mmq_dst {
++ void * p;
++ ggml_type type;
++};
++
+ typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);
+ typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);
+ typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,
+@@ -429,8 +440,8 @@
+ #include "mmq-load-tiles.cuh"
+ #include "mmq-vec-dot.cuh"
+
+-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(
+- const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,
++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(
++ const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, dst_t * __restrict__ dst,
+ const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
+@@ -456,21 +467,21 @@
+
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (y_scale_used) {
+- dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
+ } else {
+- dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
+ }
+ } else {
+- dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
+ GGML_UNUSED(y_scale_used);
+ }
+ }
+ }
+ }
+
+-template<ggml_type type, int J, bool fallback, ggml_prec prec_src1>
++template<ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float>
+ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
+- const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst,
++ const float * __restrict__ sum, const int * __restrict__ ids_dst, dst_t * __restrict__ dst,
+ const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
+
+ #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+@@ -506,12 +517,12 @@
+
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (y_scale_used) {
+- dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l];
++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]);
+ } else {
+- dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];
++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);
+ }
+ } else {
+- dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];
++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);
+ GGML_UNUSED(y_scale_used);
+ }
+ }
+@@ -869,10 +880,10 @@
+
+ // ---------------------------------------------------------------------------------------------
+
+-template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1>
++template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1, typename dst_t>
+ static __device__ __forceinline__ void mul_mat_q_process_tile(
+ const char * __restrict__ x, const int offset_x, const int * __restrict__ y,
+- const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup,
++ const int * __restrict__ ids_dst, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,
+ const float * __restrict__ y_scale,
+ const int stride_row_x, const int ncols_y, const int stride_col_dst,
+ const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {
+@@ -942,19 +953,32 @@
+
+ if (fixup) {
+ write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);
+- } else {
++ } else if constexpr (std::is_same_v<dst_t, float>) {
+ write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
++ } else if constexpr (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) {
++ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
++ } else {
++ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
+ }
+ }
+
+
++// A flat MoE grid runs only the (expert, column tile) pairs listed in tile_map, see ggml_cuda_launch_mm_ids_tile_map.
++static __device__ __forceinline__ void mmq_flat_tile(const int2 * __restrict__ tile_map, int & zt, int & jt) {
++ if (tile_map) {
++ const int2 t = tile_map[jt];
++ zt = t.x;
++ jt = t.y;
++ }
++}
++
+ // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598
+
+-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t>
+ __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1))
+ static __global__ void mul_mat_q(
+ const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,
+- const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup,
++ const int32_t * __restrict__ expert_bounds, const int2 * __restrict__ tile_map, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,
+ const float * __restrict__ y_scale,
+ const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,
+ const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
+@@ -993,9 +1017,10 @@
+ if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {
+ const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
+ const int wt = tmp2.x;
+- const int zt = tmp2.y;
+- const int jt = blockIdx.y;
++ int zt = tmp2.y;
++ int jt = blockIdx.y;
+ const int it = blockIdx.x;
++ mmq_flat_tile(tile_map, zt, jt);
+
+ // Defaults for regular matrix multiplication:
+ int col_low = 0;
+@@ -1076,14 +1101,15 @@
+ while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) {
+ int tmp = fastdiv(kbc, blocks_per_ne00);
+ uint2 tmp2 = fast_div_modulo(tmp, ntx);
+- const int jt = tmp2.y;
++ int jt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nchannels_y);
+- const int zt = tmp2.y;
++ int zt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nsamples_y);
+ const int wt = tmp2.y;
+ const int it = tmp2.x;
++ mmq_flat_tile(tile_map, zt, jt);
+
+ // Defaults for regular matrix multiplication:
+ int col_low = 0;
+@@ -1165,14 +1191,15 @@
+
+ int tmp = fastdiv(kbc, blocks_per_ne00);
+ uint2 tmp2 = fast_div_modulo(tmp, ntx);
+- const int jt = tmp2.y;
++ int jt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nchannels_y);
+- const int zt = tmp2.y;
++ int zt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nsamples_y);
+ const int wt = tmp2.y;
+ const int it = tmp2.x;
++ mmq_flat_tile(tile_map, zt, jt);
+
+ // Defaults for regular matrix multiplication:
+ int col_low = 0;
+@@ -1375,14 +1402,26 @@
+ }
+ }
+
++// FreeToken: copies an fp32 result with dst's layout into dst
++template <typename dst_t>
++static __global__ void mmq_store_dst(const float * __restrict__ src, dst_t * __restrict__ dst, const int nrows, const int stride) {
++ const int64_t col = blockIdx.x;
++ const int row = blockIdx.y*blockDim.x + threadIdx.x;
++ if (row < nrows) {
++ dst[col*stride + row] = mmq_cast<dst_t>(src[col*stride + row]);
++ }
++}
++
+ struct mmq_args {
+- const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst;
++ const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; mmq_dst dst;
+ const float * y_scale;
+ int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst;
+ int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst;
+ int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;
+ int64_t ncols_max;
+ int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance.
++ const int2 * tile_map = nullptr; // Flat MoE grid: the (expert, column tile) of each of the ntx_flat grid columns.
++ int ntx_flat = 0;
+ };
+
+ static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {
+@@ -1406,12 +1445,11 @@
+
+ const dim3 block_dims(warp_size, nwarps, 1);
+
+- CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1>), nbytes_shared);
+- CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1>), nbytes_shared);
+-
++ // a flat MoE grid puts all experts into one channel of ntx_flat column tiles
++ const int64_t nchannels_grid = args.tile_map ? 1 : args.nchannels_y;
+ const int nty = (args.nrows_x + config.I - 1) / config.I;
+- const int ntx = (args.ncols_max + config.J - 1) / config.J;
+- const int ntzw = args.nchannels_y * args.nsamples_y;
++ const int ntx = args.tile_map ? args.ntx_flat : (args.ncols_max + config.J - 1) / config.J;
++ const int ntzw = nchannels_grid * args.nsamples_y;
+ const dim3 block_nums_xy_tiling(nty, ntx, ntzw);
+
+ GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);
+@@ -1421,18 +1459,71 @@
+
+ const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);
+ const uint3 ntx_fd = init_fastdiv_values(ntx);
+- const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y);
++ const uint3 nchannels_y_fd = init_fastdiv_values(nchannels_grid);
+ const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y);
+ const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio);
+ const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);
+
+- if (!config.stream_k) {
+- mul_mat_q<type, J, fallback, prec_src1><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
+- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale,
++ const auto run = [&](auto * dst, const dim3 & block_nums, float * tmp_fixup) {
++ using dst_t = std::remove_pointer_t<decltype(dst)>;
++ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1, dst_t>), nbytes_shared);
++ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1, dst_t>), nbytes_shared);
++ mul_mat_q<type, J, fallback, prec_src1, dst_t><<<block_nums, block_dims, nbytes_shared, stream>>>
++ (args.x, args.y, args.ids_dst, args.expert_bounds, args.tile_map, dst, tmp_fixup, args.y_scale,
+ blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
+ channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
+ sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
+ ntx_fd);
++ };
++
++ // FreeToken: the fixup kernel adds partial sums into an fp32 dst, so a bf16 dst that needs it is computed in fp32 first.
++ // An fp16 dst always is, unless GGML_MMQ_F16_DST builds the fp16 kernels.
++ const auto launch = [&](const dim3 & block_nums, const bool fixup, float * tmp_fixup) {
++ bool via_f32 = fixup && args.dst.type != GGML_TYPE_F32;
++#ifndef GGML_MMQ_F16_DST
++ via_f32 = via_f32 || args.dst.type == GGML_TYPE_F16;
++#endif // GGML_MMQ_F16_DST
++ ggml_cuda_pool_alloc<float> dst_f32(ctx.pool(id));
++ float * dst_float = args.dst.type == GGML_TYPE_F32 ? (float *) args.dst.p : nullptr;
++ if (via_f32) {
++ GGML_ASSERT(args.ids_dst || (args.nchannels_y == 1 && args.nsamples_y == 1));
++ dst_float = dst_f32.alloc(args.ncols_dst * args.nrows_dst);
++ }
++
++ if (dst_float) {
++ run(dst_float, block_nums, tmp_fixup);
++ } else if (args.dst.type == GGML_TYPE_BF16) {
++ run((nv_bfloat16 *) args.dst.p, block_nums, tmp_fixup);
++ }
++#ifdef GGML_MMQ_F16_DST
++ else {
++ run((half *) args.dst.p, block_nums, tmp_fixup);
++ }
++#endif // GGML_MMQ_F16_DST
++
++ if (fixup) {
++ CUDA_CHECK(cudaGetLastError());
++ const dim3 block_nums_fixup(block_nums.x, config.I/warp_size, 1);
++ const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
++ mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
++ (args.ids_dst, args.expert_bounds, dst_float, tmp_fixup, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
++ args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,
++ ntx_fd);
++ }
++
++ if (via_f32) {
++ CUDA_CHECK(cudaGetLastError());
++ const dim3 block_nums_store(args.ncols_dst, (args.nrows_x + 255) / 256, 1);
++ if (args.dst.type == GGML_TYPE_BF16) {
++ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (nv_bfloat16 *) args.dst.p, args.nrows_x, args.nrows_dst);
++ } else {
++ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (half *) args.dst.p, args.nrows_x, args.nrows_dst);
++ }
++ }
++ };
++
++ if (!config.stream_k) {
++ launch(block_nums_xy_tiling, false, nullptr);
+ return;
+ }
+
+@@ -1441,7 +1532,8 @@
+ const int ntiles_dst = ntx * nty * ntzw;
+ const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;
+ const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);
+- const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1);
++ // The fixup kernel does not read tile_map, so a flat MoE grid always runs one tile per block.
++ const dim3 block_nums_stream_k(args.tile_map || (GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90) ? ntiles_dst : nsm, 1, 1);
+
+ GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.
+
+@@ -1453,25 +1545,7 @@
+ tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);
+ }
+
+- const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1);
+- const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
+-
+- mul_mat_q<type, J, fallback, prec_src1><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
+- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale,
+- blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
+- channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
+- sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
+- ntx_fd);
+-
+- if (!fixup_needed) {
+- return;
+- }
+-
+- CUDA_CHECK(cudaGetLastError());
+- mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
+- (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
+- args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,
+- ntx_fd);
++ launch(block_nums_stream_k, fixup_needed, tmp_fixup.ptr);
+ }
+
+ template <ggml_type type, bool fallback, ggml_prec prec_src1>
+@@ -1579,7 +1653,9 @@
+
+ // -------------------------------------------------------------------------------------------------------------------------
+
++// FreeToken: under ids, ncols_max > 0 bounds the tokens any one expert gets and sizes the tile grid; otherwise the grid is flat, see ggml_cuda_launch_mm_ids_tile_map
+ void ggml_cuda_mul_mat_q(
+- ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);
++ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
++ int64_t ncols_max = -1);
+
+ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);
+--- a/ggml/src/ggml-cuda/mmvq.cu
++++ b/ggml/src/ggml-cuda/mmvq.cu
+@@ -1,4 +1,5 @@
+ #include "mmvq.cuh"
++#include "convert.cuh"
+ #include "quantize.cuh"
+ #include "unary.cuh"
+ #include "vecdotq.cuh"
+@@ -596,10 +597,11 @@
+ return 1;
+ }
+
+-template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false>
++// FreeToken: dst_t is float, nv_bfloat16 or half
++template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false, typename dst_t = float>
+ __launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1)
+ static __global__ void mul_mat_vec_q(
+- const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
++ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst_ptr,
+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,
+ uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
+ const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
+@@ -608,7 +610,7 @@
+ const void * GGML_CUDA_RESTRICT vx = vx_ptr;
+ const void * GGML_CUDA_RESTRICT vy = vy_ptr;
+ const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
+- float * GGML_CUDA_RESTRICT dst = dst_ptr;
++ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;
+
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int qi = ggml_cuda_type_traits<type>::qi;
+@@ -629,7 +631,9 @@
+ const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y;
+ if (shared_expert) {
+ vx = fusion.shared_up;
+- dst = fusion.shared_dst;
++ if constexpr (std::is_same_v<dst_t, float>) {
++ dst = fusion.shared_dst;
++ }
+ stride_col_dst = fusion.shared_stride_col_dst;
+ }
+
+@@ -837,7 +841,7 @@
+ }
+ }
+ }
+- dst[j*stride_col_dst + i] = result;
++ dst[j*stride_col_dst + i] = ggml_cuda_cast<dst_t>(result);
+ }
+ }
+ }
+@@ -854,11 +858,11 @@
+ // Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst)
+ // Block: (warp_size, ncols_dst) - each warp handles one token independently.
+ // No shared memory reduction needed since each warp works alone.
+-template <ggml_type type, int c_rows_per_block, bool has_fusion = false>
++template <ggml_type type, int c_rows_per_block, bool has_fusion = false, typename dst_t = float>
+ __launch_bounds__(get_mmvq_mmid_max_batch_for_device<type>()*ggml_cuda_get_physical_warp_size(), 1)
+ static __global__ void mul_mat_vec_q_moe(
+ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion,
+- float * dst_ptr,
++ dst_t * dst_ptr,
+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+ const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst,
+ const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
+@@ -866,7 +870,7 @@
+ const void * GGML_CUDA_RESTRICT vx = vx_ptr;
+ const void * GGML_CUDA_RESTRICT vy = vy_ptr;
+ const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
+- float * GGML_CUDA_RESTRICT dst = dst_ptr;
++ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;
+
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int qi = ggml_cuda_type_traits<type>::qi;
+@@ -878,7 +882,9 @@
+ const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1;
+ if (shared_expert) {
+ vx = fusion.shared_up;
+- dst = fusion.shared_dst;
++ if constexpr (std::is_same_v<dst_t, float>) {
++ dst = fusion.shared_dst;
++ }
+ stride_col_dst = fusion.shared_stride_col_dst;
+ }
+
+@@ -998,7 +1004,7 @@
+ }
+ }
+ }
+- dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = result;
++ dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = ggml_cuda_cast<dst_t>(result);
+ }
+
+ if constexpr (!has_fusion) {
+@@ -1020,9 +1026,9 @@
+ return {block_nums, block_dims};
+ }
+
+-template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false>
++template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false, typename dst_t>
+ static void mul_mat_vec_q_switch_fusion(
+- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
++ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,
+ const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
+ const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
+@@ -1035,7 +1041,7 @@
+ if constexpr (c_ncols_dst == 1) {
+ if (has_fusion) {
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
+- ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters>, launch_params,
++ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters, dst_t>, launch_params,
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
+@@ -1046,15 +1052,15 @@
+ GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1");
+
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
+- ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters>, launch_params,
++ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters, dst_t>, launch_params,
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
+ }
+
+-template <ggml_type type>
++template <ggml_type type, typename dst_t>
+ static void mul_mat_vec_q_moe_launch(
+- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
++ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+ const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,
+ const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
+@@ -1071,13 +1077,13 @@
+ fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
+
+ if (has_fusion) {
+- ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true>, launch_params,
++ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true, dst_t>, launch_params,
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
+ stride_row_x, stride_col_y, stride_col_dst,
+ stride_channel_x, stride_channel_y, stride_channel_dst,
+ ncols_dst, ids_stride);
+ } else {
+- ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false>, launch_params,
++ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false, dst_t>, launch_params,
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
+ stride_row_x, stride_col_y, stride_col_dst,
+ stride_channel_x, stride_channel_y, stride_channel_dst,
+@@ -1085,9 +1091,9 @@
+ }
+ }
+
+-template <ggml_type type>
++template <ggml_type type, typename dst_t>
+ static void mul_mat_vec_q_switch_ncols_dst(
+- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
++ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
+ const int ncols_x, const int nrows_x, const int ncols_dst,
+ const int stride_row_x, const int stride_col_y, const int stride_col_dst,
+ const int nchannels_x, const int nchannels_y, const int nchannels_dst,
+@@ -1277,8 +1283,10 @@
+ GGML_ABORT("fatal error");
+ }
+ }
++
++template <typename dst_t>
+ static void mul_mat_vec_q_switch_type(
+- const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
++ const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
+ const int ncols_x, const int nrows_x, const int ncols_dst,
+ const int stride_row_x, const int stride_col_y, const int stride_col_dst,
+ const int nchannels_x, const int nchannels_y, const int nchannels_dst,
+@@ -1431,9 +1439,10 @@
+
+ void ggml_cuda_mul_mat_vec_q(
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
+- const ggml_cuda_mm_fusion_args_host * fusion) {
+- GGML_ASSERT( src1->type == GGML_TYPE_F32);
+- GGML_ASSERT( dst->type == GGML_TYPE_F32);
++ const ggml_cuda_mm_fusion_args_host * fusion, const void * src1_q8_1_pre) {
++ // FreeToken: activations and results may also be bf16 or fp16
++ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);
++ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);
+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+@@ -1526,12 +1535,22 @@
+ }
+
+ const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);
+- ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);
+- {
++ // FreeToken: src1_q8_1_pre is src1 already quantized by quantize_row_q8_1_cuda with this padding and layout
++ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool());
++ const void * src1_q8 = src1_q8_1_pre;
++ if (src1_q8 == nullptr) {
++ src1_q8_1.alloc(ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);
++ src1_q8 = src1_q8_1.get();
+ const int64_t s11 = src1->nb[1] / ts_src1;
+ const int64_t s12 = src1->nb[2] / ts_src1;
+ const int64_t s13 = src1->nb[3] / ts_src1;
+- quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
++ if (src1->type == GGML_TYPE_BF16) {
++ quantize_row_q8_1_cuda((const nv_bfloat16 *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
++ } else if (src1->type == GGML_TYPE_F16) {
++ quantize_row_q8_1_cuda((const half *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
++ } else {
++ quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
++ }
+ }
+
+ const int64_t s01 = src0->nb[1] / ts_src0;
+@@ -1556,11 +1575,23 @@
+
+ const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
+
+- mul_mat_vec_q_switch_type(
+- src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00,
+- ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
+- ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
+- ne03, ne3, s03, s13, s3, ids_stride, stream);
++ const auto launch = [&](auto * dst_ptr) {
++ mul_mat_vec_q_switch_type(
++ src0->data, src0->type, src1_q8, ids_d, fusion_local, dst_ptr, ne00,
++ ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
++ ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
++ ne03, ne3, s03, s13, s3, ids_stride, stream);
++ };
++ if (dst->type == GGML_TYPE_BF16) {
++ // FreeToken: the fused shared expert writes an fp32 shared_dst
++ GGML_ASSERT(!fusion || !fusion->shared_up);
++ launch((nv_bfloat16 *) dst->data);
++ } else if (dst->type == GGML_TYPE_F16) {
++ GGML_ASSERT(!fusion || !fusion->shared_up);
++ launch((half *) dst->data);
++ } else {
++ launch(dst_d);
++ }
+ }
+
+ void ggml_cuda_op_mul_mat_vec_q(
+--- a/ggml/src/ggml-cuda/mmvq.cuh
++++ b/ggml/src/ggml-cuda/mmvq.cuh
+@@ -8,8 +8,10 @@
+ // based on the quantization type and GPU architecture (compute capability).
+ int get_mmvq_mmid_max_batch(ggml_type type, int cc);
+
++// FreeToken: src1_q8_1_pre skips the internal quantization of src1
+ void ggml_cuda_mul_mat_vec_q(ggml_backend_cuda_context & ctx,
+- const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr);
++ const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr,
++ const void * src1_q8_1_pre = nullptr);
+
+ void ggml_cuda_op_mul_mat_vec_q(
+ ggml_backend_cuda_context & ctx,
+--- a/ggml/src/ggml-cuda/quantize.cu
++++ b/ggml/src/ggml-cuda/quantize.cu
+@@ -1,4 +1,5 @@
+ #include "quantize.cuh"
++#include "convert.cuh"
+ #include <cstdint>
+
+ #if defined(BLACKWELL_MMA_AVAILABLE)
+@@ -50,13 +51,14 @@
+ #endif // CUDART_VERSION >= 12080
+ #endif // defined(BLACKWELL_MMA_AVAILABLE)
+
++template <typename src_t>
+ __launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1)
+ static __global__ void quantize_q8_1(
+- const float * x_ptr, void * vy_ptr,
++ const src_t * x_ptr, void * vy_ptr,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const uint32_t ne1, const uint3 ne2) {
+ ggml_cuda_pdl_lc();
+- const float * GGML_CUDA_RESTRICT x = x_ptr;
++ const src_t * GGML_CUDA_RESTRICT x = x_ptr;
+ void * GGML_CUDA_RESTRICT vy = vy_ptr;
+ const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
+
+@@ -81,7 +83,7 @@
+ const int64_t iqs = i_cont % QK8_1; // quant index
+
+ ggml_cuda_pdl_sync();
+- const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f;
++ const float xi = i0 < ne00 ? ggml_cuda_cast<float>(x[i03*s03 + i02*s02 + i01*s01 + i00]) : 0.0f;
+ float amax = fabsf(xi);
+ float sum = xi;
+
+@@ -452,10 +454,29 @@
+ GGML_UNUSED(n_expert_used);
+ }
+
++// FreeToken: 4 consecutive activations as float, from fp32, bf16 or fp16
++static __device__ __forceinline__ float4 load_float4(const float * x) {
++ return *(const float4 *) x;
++}
++
++static __device__ __forceinline__ float4 load_float4(const nv_bfloat16 * x) {
++ const uint2 v = *(const uint2 *) x;
++ const float2 lo = __bfloat1622float2(*(const nv_bfloat162 *) &v.x);
++ const float2 hi = __bfloat1622float2(*(const nv_bfloat162 *) &v.y);
++ return make_float4(lo.x, lo.y, hi.x, hi.y);
++}
++
++static __device__ __forceinline__ float4 load_float4(const half * x) {
++ const uint2 v = *(const uint2 *) x;
++ const float2 lo = __half22float2(*(const half2 *) &v.x);
++ const float2 hi = __half22float2(*(const half2 *) &v.y);
++ return make_float4(lo.x, lo.y, hi.x, hi.y);
++}
++
+ // scatter: grid over tokens, quantize once, write to all the token's compact rows
+-template <mmq_q8_1_ds_layout ds_layout, bool scatter>
++template <mmq_q8_1_ds_layout ds_layout, bool scatter, typename src_t>
+ static __global__ void quantize_mmq_q8_1(
+- const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,
++ const src_t * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) {
+
+@@ -481,14 +502,13 @@
+ base_idx = i3*s03 + i2*s02 + i01*s01;
+ }
+
+- const float4 * x4 = (const float4 *) x;
+ block_q8_1_mmq * y = (block_q8_1_mmq *) vy;
+
+ const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel
+ const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block
+
+ // Load 4 floats per thread and calculate max. abs. value between them:
+- const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
++ const float4 xi = i0 < ne00 ? load_float4(x + base_idx + i00) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ float amax = fabsf(xi.x);
+ amax = fmaxf(amax, fabsf(xi.y));
+ amax = fmaxf(amax, fabsf(xi.z));
+@@ -554,8 +574,9 @@
+ GGML_UNUSED(n_expert_used);
+ }
+
++template <typename src_t>
+ void quantize_row_q8_1_cuda(
+- const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,
++ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {
+ GGML_ASSERT(!ids);
+@@ -567,12 +588,13 @@
+ const dim3 num_blocks(block_num_x, ne1, ne2*ne3);
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream);
+- ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);
++ ggml_cuda_kernel_launch(quantize_q8_1<src_t>, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);
+ GGML_UNUSED(type_src0);
+ }
+
++template <typename src_t>
+ void quantize_mmq_q8_1_cuda(
+- const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,
++ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {
+ GGML_ASSERT(ne00 % 4 == 0);
+@@ -584,15 +606,15 @@
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+ switch (mmq_get_q8_1_ds_layout(type_src0)) {
+ case MMQ_Q8_1_DS_LAYOUT_D4:
+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, false>
++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, false, src_t>
+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_DS4:
+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, false>
++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, false, src_t>
+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_D2S6:
+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, false>
++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, false, src_t>
+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ break;
+ default:
+@@ -601,8 +623,9 @@
+ }
+
+ // scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)
++template <typename src_t>
+ void quantize_scatter_mmq_q8_1_cuda(
+- const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
++ const src_t * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
+ const int64_t ne00, const int64_t stride_token, const int64_t ne0,
+ const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) {
+ GGML_ASSERT(ne00 % 4 == 0);
+@@ -613,15 +636,15 @@
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+ switch (mmq_get_q8_1_ds_layout(type_src0)) {
+ case MMQ_Q8_1_DS_LAYOUT_D4:
+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, true><<<num_blocks, block_size, 0, stream>>>(
++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, true, src_t><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_DS4:
+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, true><<<num_blocks, block_size, 0, stream>>>(
++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, true, src_t><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_D2S6:
+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, true><<<num_blocks, block_size, 0, stream>>>(
++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, true, src_t><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
+ break;
+ default:
+@@ -692,3 +715,15 @@
+ quantize_mmq_mxfp4<false><<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ }
+ }
++
++#define FREETOKEN_QUANTIZE_INSTANCES(src_t) \
++ template void quantize_row_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \
++ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \
++ template void quantize_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \
++ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \
++ template void quantize_scatter_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, \
++ int64_t, int64_t, int64_t, int64_t, int, cudaStream_t);
++
++FREETOKEN_QUANTIZE_INSTANCES(float)
++FREETOKEN_QUANTIZE_INSTANCES(nv_bfloat16)
++FREETOKEN_QUANTIZE_INSTANCES(half)
+--- a/ggml/src/ggml-cuda/quantize.cuh
++++ b/ggml/src/ggml-cuda/quantize.cuh
+@@ -16,13 +16,16 @@
+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
+
++// FreeToken: src_t is float, nv_bfloat16 or half
++template <typename src_t>
+ void quantize_row_q8_1_cuda(
+- const float * x, const int32_t * ids, void * vy,
++ const src_t * x, const int32_t * ids, void * vy,
+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
+
++template <typename src_t>
+ void quantize_mmq_q8_1_cuda(
+- const float * x, const int32_t * ids, void * vy,
++ const src_t * x, const int32_t * ids, void * vy,
+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
+
+@@ -57,7 +60,8 @@
+ int n_expert_used,
+ cudaStream_t stream);
+
+-void quantize_scatter_mmq_q8_1_cuda(const float * x,
++template <typename src_t>
++void quantize_scatter_mmq_q8_1_cuda(const src_t * x,
+ const int32_t * ids_src1_inv,
+ void * vy,
+ ggml_type type_src0,
diff --git a/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cu b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cu
new file mode 100644
index 00000000..cc735c3b
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cu
@@ -0,0 +1,426 @@
+// tvm-ffi ops over the vendored ggml-cuda kernels. Weights stay in their packed ggml blocks.
+// Activations go through the same q8_1 quantization and kernel choice as llama.cpp's mul_mat / mul_mat_id.
+// Activations are fp32, bf16 or fp16 with contiguous rows. Results have the activation dtype and are accumulated in fp32; the caller allocates every output.
+
+#include <freetoken/utils.h>
+
+#include "common.cuh"
+#include "convert.cuh"
+#include "getrows.cuh"
+#include "mmq.cuh"
+#include "mmvq.cuh"
+#include "quantize.cuh"
+
+#include <tvm/ffi/container/tensor.h>
+#include <tvm/ffi/extra/c_env_api.h>
+#include <tvm/ffi/function.h>
+#include <tvm/ffi/optional.h>
+#include <tvm/ffi/string.h>
+
+#include <algorithm>
+#include <array>
+#include <memory>
+#include <string>
+
+using tvm::ffi::Optional;
+using tvm::ffi::String;
+using tvm::ffi::TensorView;
+
+namespace {
+
+using host::RuntimeCheck;
+
+struct DeviceGuard {
+ int prev, device;
+
+ explicit DeviceGuard(DLDevice d) : prev(ggml_cuda_get_device()), device(d.device_id) {
+#ifdef GGML_USE_HIP
+ RuntimeCheck(d.device_type == kDLROCM, "ggml ops take ROCm tensors");
+#else
+ RuntimeCheck(d.device_type == kDLCUDA, "ggml ops take CUDA tensors");
+#endif
+ ggml_cuda_set_device(device);
+ }
+
+ ~DeviceGuard() {
+ if (prev != device) {
+ cudaSetDevice(prev);
+ }
+ }
+};
+
+// ggml treats a null stream as unset and would create its own; name the default stream explicitly
+cudaStream_t current_stream(DLDevice d) {
+ auto stream = static_cast<cudaStream_t>(TVMFFIEnvGetStream(d.device_type, d.device_id));
+#ifdef GGML_USE_HIP
+ return stream == nullptr ? hipStreamLegacy : stream;
+#else
+ return stream == nullptr ? cudaStreamLegacy : stream;
+#endif
+}
+
+ggml_backend_cuda_context & context(DLDevice d) {
+ static std::unique_ptr<ggml_backend_cuda_context> ctxs[GGML_CUDA_MAX_DEVICES];
+ const int dev = d.device_id;
+ if (!ctxs[dev]) {
+ ctxs[dev] = std::make_unique<ggml_backend_cuda_context>(dev);
+ }
+ ctxs[dev]->curr_stream_no = 0;
+ ctxs[dev]->streams[dev][0] = current_stream(d);
+ return *ctxs[dev];
+}
+
+ggml_tensor describe(ggml_type type, const void * data, std::array<int64_t, 4> ne, std::array<size_t, 4> nb) {
+ ggml_tensor t{};
+ t.type = type;
+ t.data = const_cast<void *>(data);
+ for (int i = 0; i < 4; ++i) {
+ t.ne[i] = ne[i];
+ t.nb[i] = nb[i];
+ }
+ return t;
+}
+
+int device_cc() { return ggml_cuda_info().devices[ggml_cuda_get_device()].cc; }
+
+bool is_dtype(DLDataType dt, DLDataTypeCode code, int bits) { return dt.code == code && dt.bits == bits && dt.lanes == 1; }
+
+ggml_type float_type(TensorView t) {
+ const DLDataType dt = t.dtype();
+ if (is_dtype(dt, kDLFloat, 32)) {
+ return GGML_TYPE_F32;
+ }
+ if (is_dtype(dt, kDLBfloat, 16)) {
+ return GGML_TYPE_BF16;
+ }
+ RuntimeCheck(is_dtype(dt, kDLFloat, 16), "ggml tensors must be fp32, bf16 or fp16");
+ return GGML_TYPE_F16;
+}
+
+// the activation dtype, which the result must share
+ggml_type io_type(TensorView x, TensorView out) {
+ const ggml_type type = float_type(x);
+ RuntimeCheck(float_type(out) == type, "the result must have the activation dtype");
+ return type;
+}
+
+int64_t check_weight(TensorView w, ggml_type type, int64_t k) {
+ RuntimeCheck(is_dtype(w.dtype(), kDLUInt, 8), "ggml weight must be a uint8 tensor");
+ RuntimeCheck(w.stride(w.ndim() - 1) == 1, "ggml weight rows must be contiguous");
+ const int64_t bs = ggml_blck_size(type);
+ RuntimeCheck(k % bs == 0, "K=", k, " is not a multiple of the ", ggml_type_name(type), " block size ", bs);
+ const int64_t row_bytes = k / bs * (int64_t) ggml_type_size(type);
+ RuntimeCheck(w.size(w.ndim() - 1) == row_bytes, "ggml weight row is ", w.size(w.ndim() - 1), " bytes, ", ggml_type_name(type),
+ " K=", k, " needs ", row_bytes);
+ return row_bytes;
+}
+
+int64_t q8_1_row_bytes(int64_t k) { return GGML_PAD(k, MATRIX_ROW_PADDING) / QK8_1 * (int64_t) sizeof(block_q8_1); }
+
+// MMQ tiles read up to MATRIX_ROW_PADDING values past the last row; the caller keeps this many bytes after it
+int64_t mmq_tail_bytes(int64_t type_id, int64_t k) {
+ const int64_t rem = k % MATRIX_ROW_PADDING;
+ return rem == 0 ? 0 : (int64_t) ggml_row_size((ggml_type) type_id, MATRIX_ROW_PADDING - rem);
+}
+
+struct Activation {
+ const void * data;
+ ggml_type type;
+ size_t esz;
+};
+
+// The native fp4 MMQ path quantizes fp32 activations only; it is built only with GGML_CUDA_FP4_MMA.
+bool mmq_reads_16bit(ggml_type type) {
+#ifdef GGML_CUDA_FP4_MMA
+ return type != GGML_TYPE_MXFP4 && type != GGML_TYPE_NVFP4;
+#else
+ GGML_UNUSED(type);
+ return true;
+#endif
+}
+
+// x as the kernel reads it: x itself, or an fp32 copy in ``tmp`` for a kernel that reads fp32 only
+Activation activation(TensorView x, bool f16_ok, ggml_cuda_pool_alloc<float> & tmp, cudaStream_t stream) {
+ RuntimeCheck(x.IsContiguous(), "ggml activations must be contiguous");
+ const ggml_type type = float_type(x);
+ if (type == GGML_TYPE_F32) {
+ return {x.data_ptr(), type, 4};
+ }
+ if (f16_ok) {
+ return {x.data_ptr(), type, 2};
+ }
+ float * f32 = tmp.alloc(x.numel());
+ ggml_get_to_fp32_cuda(type)(x.data_ptr(), f32, x.numel(), stream);
+ return {f32, GGML_TYPE_F32, 4};
+}
+
+} // namespace
+
+// out[M, q8_1_row_bytes(K)] = x[M, K] as MMVQ's q8_1 blocks, for GEMVs that share one input
+void op_quantize_q8_1(TensorView x, TensorView out) {
+ RuntimeCheck(x.ndim() == 2 && x.stride(1) == 1, "quantize_q8_1 takes x[M, K] with contiguous rows");
+ const DeviceGuard guard(x.device());
+ const int64_t m = x.size(0), k = x.size(1), k_padded = GGML_PAD(k, MATRIX_ROW_PADDING);
+ RuntimeCheck(is_dtype(out.dtype(), kDLUInt, 8) && out.IsContiguous() && out.ndim() == 2 && out.size(0) == m &&
+ out.size(1) == q8_1_row_bytes(k),
+ "out must be a contiguous uint8 [M, q8_1_row_bytes(K)] tensor");
+ if (m == 0) {
+ return;
+ }
+ const int64_t s1 = x.stride(0);
+ const cudaStream_t stream = current_stream(x.device());
+ const ggml_type type = float_type(x);
+ if (type == GGML_TYPE_BF16) {
+ quantize_row_q8_1_cuda((const nv_bfloat16 *) x.data_ptr(), nullptr, out.data_ptr(), GGML_TYPE_Q8_0, k, s1, s1 * m, s1 * m, k_padded, m, 1, 1, stream);
+ } else if (type == GGML_TYPE_F16) {
+ quantize_row_q8_1_cuda((const half *) x.data_ptr(), nullptr, out.data_ptr(), GGML_TYPE_Q8_0, k, s1, s1 * m, s1 * m, k_padded, m, 1, 1, stream);
+ } else {
+ quantize_row_q8_1_cuda((const float *) x.data_ptr(), nullptr, out.data_ptr(), GGML_TYPE_Q8_0, k, s1, s1 * m, s1 * m, k_padded, m, 1, 1, stream);
+ }
+}
+
+// out[M, N] = x[M, K] @ W[N, K]^T; out may be a view with strided rows; MMVQ reads ``xq`` (op_quantize_q8_1 of x) instead of quantizing x again
+void op_mul_mat(TensorView w, int64_t type_id, TensorView x, TensorView out, Optional<TensorView> xq) {
+ RuntimeCheck(x.ndim() == 2 && w.ndim() == 2, "mul_mat takes w[N, row_bytes] and x[M, K]");
+ const DeviceGuard guard(x.device());
+ const ggml_type type = (ggml_type) type_id;
+ const int64_t m = x.size(0), k = x.size(1), n = w.size(0);
+ check_weight(w, type, k);
+ RuntimeCheck(out.ndim() == 2 && out.size(0) == m && out.size(1) == n && out.stride(1) == 1, "mul_mat out must be a [M, N] view with contiguous rows");
+ const ggml_type otype = io_type(x, out);
+ if (m == 0) {
+ return;
+ }
+ const size_t s1 = w.stride(0);
+ ggml_tensor src0 = describe(type, w.data_ptr(), {k, n, 1, 1}, {ggml_type_size(type), s1, s1 * n, s1 * n});
+ const size_t oe = ggml_type_size(otype), os = out.stride(0) * oe;
+ ggml_tensor dst = describe(otype, out.data_ptr(), {n, m, 1, 1}, {oe, os, os * m, os * m});
+ auto & ctx = context(x.device());
+ const cudaStream_t stream = ctx.stream();
+ const int cc = device_cc();
+ ggml_cuda_pool_alloc<float> x32(ctx.pool());
+ if (ggml_cuda_should_use_mmvq(type, cc, m)) {
+ const Activation a = activation(x, true, x32, stream);
+ ggml_tensor src1 = describe(a.type, a.data, {k, m, 1, 1}, {a.esz, a.esz * k, a.esz * k * m, a.esz * k * m});
+ const void * q8 = nullptr;
+ if (xq.has_value()) {
+ const TensorView q = xq.value();
+ RuntimeCheck(is_dtype(q.dtype(), kDLUInt, 8) && q.IsContiguous() && q.ndim() == 2 && q.size(0) == m && q.size(1) == q8_1_row_bytes(k),
+ "xq must be quantize_q8_1 of x");
+ q8 = q.data_ptr();
+ }
+ ggml_cuda_mul_mat_vec_q(ctx, &src0, &src1, nullptr, &dst, nullptr, q8);
+ } else if (ggml_cuda_should_use_mmq(type, cc, m, 0)) {
+ const Activation a = activation(x, mmq_reads_16bit(type), x32, stream);
+ ggml_tensor src1 = describe(a.type, a.data, {k, m, 1, 1}, {a.esz, a.esz * k, a.esz * k * m, a.esz * k * m});
+ ggml_cuda_mul_mat_q(ctx, &src0, &src1, nullptr, &dst);
+ } else {
+ RuntimeCheck(false, "no ggml kernel for ", ggml_type_name(type), " with M=", m);
+ }
+}
+
+// out[T, k, N]: row (t, j) multiplies x[t, j or 0] by expert ids[t, j] of W[E, N, K]; the matrix kernel runs a flat grid over the tiles that hold routes
+void op_mul_mat_id(TensorView w, int64_t type_id, TensorView x, TensorView ids, TensorView out) {
+ RuntimeCheck(w.ndim() == 3, "mul_mat_id takes w[E, N, row_bytes]");
+ RuntimeCheck(is_dtype(ids.dtype(), kDLInt, 32) && ids.ndim() == 2 && ids.IsContiguous(), "ids must be a contiguous int32 [T, k] tensor");
+ const DeviceGuard guard(x.device());
+ const ggml_type type = (ggml_type) type_id;
+ const int64_t t = ids.size(0), top_k = ids.size(1);
+ RuntimeCheck(x.ndim() == 3 && x.size(0) == t && (x.size(1) == 1 || x.size(1) == top_k), "x must be [T, 1, K] or [T, k, K]");
+ const int64_t ky = x.size(1), k = x.size(2), e = w.size(0), n = w.size(1);
+ check_weight(w, type, k);
+ RuntimeCheck(out.ndim() == 3 && out.size(0) == t && out.size(1) == top_k && out.size(2) == n && out.IsContiguous(),
+ "out must be a contiguous [T, k, N] tensor");
+ const ggml_type otype = io_type(x, out);
+ if (t == 0) {
+ return;
+ }
+ const size_t s1 = w.stride(1), s2 = w.stride(0);
+ ggml_tensor src0 = describe(type, w.data_ptr(), {k, n, e, 1}, {ggml_type_size(type), s1, s2, s2 * e});
+ const size_t ids_stride = sizeof(int32_t) * top_k, ids_bytes = ids_stride * t;
+ ggml_tensor tids = describe(GGML_TYPE_I32, ids.data_ptr(), {top_k, t, 1, 1}, {sizeof(int32_t), ids_stride, ids_bytes, ids_bytes});
+ auto & ctx = context(x.device());
+ const cudaStream_t stream = ctx.stream();
+ const int cc = device_cc();
+ const bool mmvq = t <= MMVQ_MAX_BATCH_SIZE && t <= get_mmvq_mmid_max_batch(type, cc);
+ RuntimeCheck(mmvq || ggml_cuda_should_use_mmq(type, cc, t, e), "no ggml MoE kernel for ", ggml_type_name(type), " with T=", t);
+ ggml_cuda_pool_alloc<float> x32(ctx.pool());
+ const Activation a = activation(x, mmvq || mmq_reads_16bit(type), x32, stream);
+ ggml_tensor src1 = describe(a.type, a.data, {k, ky, t, 1}, {a.esz, a.esz * k, a.esz * k * ky, a.esz * k * ky * t});
+ const size_t oe = ggml_type_size(otype);
+ ggml_tensor dst = describe(otype, out.data_ptr(), {n, top_k, t, 1}, {oe, oe * n, oe * n * top_k, oe * n * top_k * t});
+ if (mmvq) {
+ ggml_cuda_mul_mat_vec_q(ctx, &src0, &src1, &tids, &dst);
+ } else {
+ ggml_cuda_mul_mat_q(ctx, &src0, &src1, &tids, &dst, 0);
+ }
+}
+
+namespace {
+
+// The GLUs the MMVQ epilogue computes; any other ggml_glu_op falls through to a plain gate * up there.
+ggml_glu_op fused_glu_op(const String & name) {
+ if (name == "swiglu") {
+ return GGML_GLU_OP_SWIGLU;
+ }
+ if (name == "geglu") {
+ return GGML_GLU_OP_GEGLU;
+ }
+ if (name == "swiglu_oai") {
+ return GGML_GLU_OP_SWIGLU_OAI;
+ }
+ RuntimeCheck(name == "swiglu_clamp", "MMVQ fuses swiglu, geglu, swiglu_oai or swiglu_clamp, not ", std::string(name));
+ return GGML_GLU_OP_SWIGLU_CLAMP;
+}
+
+} // namespace
+
+// out[T, k, I] = glu(x @ gate^T, x @ up^T) for small T in one MMVQ pass; w[E, 2I, row_bytes] holds each expert's gate rows then its up rows; limit is swiglu_clamp's
+void op_mul_mat_id_glu(TensorView w, int64_t type_id, TensorView x, TensorView ids, TensorView out, String glu, double limit) {
+ const ggml_glu_op glu_op = fused_glu_op(glu);
+ RuntimeCheck(w.ndim() == 3 && w.size(1) % 2 == 0, "mul_mat_id_glu takes w[E, 2I, row_bytes]");
+ RuntimeCheck(is_dtype(ids.dtype(), kDLInt, 32) && ids.ndim() == 2 && ids.IsContiguous(), "ids must be a contiguous int32 [T, k] tensor");
+ RuntimeCheck(x.ndim() == 2 && x.size(0) == ids.size(0), "x must be [T, K]");
+ const DeviceGuard guard(x.device());
+ const ggml_type type = (ggml_type) type_id;
+ const int64_t t = ids.size(0), top_k = ids.size(1), k = x.size(1), e = w.size(0), n = w.size(1) / 2;
+ check_weight(w, type, k);
+ RuntimeCheck(out.IsContiguous() && out.ndim() == 3 && out.size(0) == t && out.size(1) == top_k && out.size(2) == n,
+ "out must be a contiguous [T, k, I] tensor");
+ const ggml_type otype = io_type(x, out);
+ const int cc = device_cc();
+ RuntimeCheck(t <= MMVQ_MAX_BATCH_SIZE && t <= get_mmvq_mmid_max_batch(type, cc), "mul_mat_id_glu is the small-batch path, T=", t);
+ if (t == 0) {
+ return;
+ }
+ auto & ctx = context(x.device());
+ ggml_cuda_pool_alloc<float> x32(ctx.pool());
+ const Activation a = activation(x, true, x32, ctx.stream());
+ const size_t s1 = w.stride(1), s2 = w.stride(0);
+ const char * base = (const char *) w.data_ptr();
+ ggml_tensor up = describe(type, base + n * s1, {k, n, e, 1}, {ggml_type_size(type), s1, s2, s2 * e});
+ ggml_tensor gate = describe(type, base, {k, n, e, 1}, {ggml_type_size(type), s1, s2, s2 * e});
+ ggml_tensor src1 = describe(a.type, a.data, {k, 1, t, 1}, {a.esz, a.esz * k, a.esz * k, a.esz * k * t});
+ const size_t ids_stride = sizeof(int32_t) * top_k, ids_bytes = ids_stride * t;
+ ggml_tensor tids = describe(GGML_TYPE_I32, ids.data_ptr(), {top_k, t, 1, 1}, {sizeof(int32_t), ids_stride, ids_bytes, ids_bytes});
+ const size_t oe = ggml_type_size(otype);
+ ggml_tensor dst = describe(otype, out.data_ptr(), {n, top_k, t, 1}, {oe, oe * n, oe * n * top_k, oe * n * top_k * t});
+ ggml_cuda_mm_fusion_args_host fusion{};
+ fusion.gate = &gate;
+ fusion.glu_op = glu_op;
+ fusion.glu_limit = (float) limit;
+ ggml_cuda_mul_mat_vec_q(ctx, &up, &src1, &tids, &dst, &fusion);
+}
+
+namespace {
+
+template <typename T>
+__global__ void moe_weighted_sum_kernel(const T * __restrict__ y, const float * __restrict__ w, T * __restrict__ out,
+ const int64_t n, const int top_k) {
+ const int64_t token = blockIdx.x;
+ const int64_t col = (int64_t) blockIdx.y * blockDim.x + threadIdx.x;
+ if (col >= n) {
+ return;
+ }
+ const int64_t row0 = token * top_k;
+ float sum = 0.0f;
+ for (int j = 0; j < top_k; ++j) {
+ sum += ggml_cuda_cast<float>(y[(row0 + j) * n + col]) * w[row0 + j];
+ }
+ out[token * n + col] = ggml_cuda_cast<T>(sum);
+}
+
+} // namespace
+
+// out[T, N] = sum_j weights[t, j] * y[t, j, :], summed in fp32
+void op_moe_weighted_sum(TensorView y, TensorView weights, TensorView out) {
+ RuntimeCheck(y.ndim() == 3 && y.IsContiguous(), "y must be a contiguous [T, k, N] tensor");
+ const int64_t t = y.size(0), top_k = y.size(1), n = y.size(2);
+ RuntimeCheck(float_type(weights) == GGML_TYPE_F32 && weights.IsContiguous() && weights.numel() == t * top_k, "weights must be a contiguous fp32 [T, k] tensor");
+ RuntimeCheck(out.ndim() == 2 && out.size(0) == t && out.size(1) == n && out.IsContiguous(), "out must be a contiguous [T, N] tensor");
+ const ggml_type otype = io_type(y, out);
+ const DeviceGuard guard(y.device());
+ if (t == 0) {
+ return;
+ }
+ constexpr int threads = 256;
+ const dim3 blocks(t, (n + threads - 1) / threads);
+ const cudaStream_t stream = current_stream(y.device());
+ const float * wp = (const float *) weights.data_ptr();
+ if (otype == GGML_TYPE_BF16) {
+ moe_weighted_sum_kernel<<<blocks, threads, 0, stream>>>((const nv_bfloat16 *) y.data_ptr(), wp, (nv_bfloat16 *) out.data_ptr(), n, top_k);
+ } else if (otype == GGML_TYPE_F16) {
+ moe_weighted_sum_kernel<<<blocks, threads, 0, stream>>>((const half *) y.data_ptr(), wp, (half *) out.data_ptr(), n, top_k);
+ } else {
+ moe_weighted_sum_kernel<<<blocks, threads, 0, stream>>>((const float *) y.data_ptr(), wp, (float *) out.data_ptr(), n, top_k);
+ }
+ CUDA_CHECK(cudaGetLastError());
+}
+
+// out[rows, K] = W[rows, row_bytes] dequantized to out's dtype
+void op_dequantize(TensorView w, int64_t type_id, int64_t k, TensorView out) {
+ const DeviceGuard guard(w.device());
+ const ggml_type type = (ggml_type) type_id;
+ check_weight(w, type, k);
+ RuntimeCheck(w.IsContiguous(), "dequantize needs a contiguous weight");
+ const int64_t rows = w.numel() / w.size(w.ndim() - 1);
+ RuntimeCheck(out.IsContiguous() && out.numel() == rows * k, "out must be a contiguous [rows, K] tensor");
+ const ggml_type otype = float_type(out);
+ if (rows == 0) {
+ return;
+ }
+ const cudaStream_t stream = current_stream(w.device());
+ if (type == otype) {
+ CUDA_CHECK(cudaMemcpyAsync(out.data_ptr(), w.data_ptr(), rows * k * ggml_type_size(type), cudaMemcpyDeviceToDevice, stream));
+ return;
+ }
+ if (otype == GGML_TYPE_BF16) {
+ const auto convert = ggml_get_to_bf16_cuda(type);
+ RuntimeCheck(convert != nullptr, "no ggml bf16 conversion for ", ggml_type_name(type));
+ convert(w.data_ptr(), (nv_bfloat16 *) out.data_ptr(), rows * k, stream);
+ } else if (otype == GGML_TYPE_F16) {
+ const auto convert = ggml_get_to_fp16_cuda(type);
+ RuntimeCheck(convert != nullptr, "no ggml fp16 conversion for ", ggml_type_name(type));
+ convert(w.data_ptr(), (half *) out.data_ptr(), rows * k, stream);
+ } else {
+ const auto convert = ggml_get_to_fp32_cuda(type);
+ RuntimeCheck(convert != nullptr, "no ggml fp32 conversion for ", ggml_type_name(type));
+ convert(w.data_ptr(), (float *) out.data_ptr(), rows * k, stream);
+ }
+}
+
+// out[i] = dequant(W[ids[i]]) for a table W[rows, row_bytes]
+void op_get_rows(TensorView w, int64_t type_id, int64_t k, TensorView ids, TensorView out) {
+ RuntimeCheck(is_dtype(ids.dtype(), kDLInt, 32) && ids.ndim() == 1 && ids.IsContiguous(), "ids must be a contiguous int32 vector");
+ RuntimeCheck(w.ndim() == 2, "get_rows takes w[rows, row_bytes]");
+ const DeviceGuard guard(w.device());
+ const ggml_type type = (ggml_type) type_id;
+ check_weight(w, type, k);
+ const int64_t n = ids.size(0);
+ RuntimeCheck(out.IsContiguous() && out.ndim() == 2 && out.size(0) == n && out.size(1) == k, "out must be a contiguous [n, K] tensor");
+ const ggml_type otype = float_type(out);
+ if (n == 0) {
+ return;
+ }
+ const size_t nb01 = w.stride(0), esz = otype == GGML_TYPE_F32 ? 4 : 2;
+ get_rows_cuda(w.data_ptr(), type, (const int32_t *) ids.data_ptr(), out.data_ptr(), otype, k, nb01, nb01 * w.size(0), nb01 * w.size(0),
+ n, 1, 1, 4, 4 * n, 4 * n, k * esz, k * esz * n, k * esz * n, current_stream(w.device()));
+}
+
+bool op_uses_mmvq(int64_t type_id, int64_t m) { return ggml_cuda_should_use_mmvq((ggml_type) type_id, device_cc(), m); }
+
+int64_t op_mmvq_max_tokens(int64_t type_id) {
+ return std::min<int64_t>(MMVQ_MAX_BATCH_SIZE, get_mmvq_mmid_max_batch((ggml_type) type_id, device_cc()));
+}
+
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(quantize_q8_1, op_quantize_q8_1);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(q8_1_row_bytes, q8_1_row_bytes);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mmq_tail_bytes, mmq_tail_bytes);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mul_mat, op_mul_mat);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mul_mat_id, op_mul_mat_id);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mul_mat_id_glu, op_mul_mat_id_glu);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(moe_weighted_sum, op_moe_weighted_sum);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(dequantize, op_dequantize);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(get_rows, op_get_rows);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(uses_mmvq, op_uses_mmvq);
+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mmvq_max_tokens, op_mmvq_max_tokens);
diff --git a/python/freetoken/kernel/csrc/ggml/freetoken/ggml_shim.cu b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_shim.cu
new file mode 100644
index 00000000..80ef7285
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_shim.cu
@@ -0,0 +1,255 @@
+// The ggml-base and ggml-cuda.cu host symbols that the vendored ggml-cuda kernels reference.
+// Tensor descriptors built by ggml_ops.cu never have a backend buffer, and temporary buffers come from tvm-ffi's env allocator (torch's caching allocator under torch).
+
+#include "common.cuh"
+
+#include <tvm/ffi/container/tensor.h>
+#include <tvm/ffi/extra/c_env_api.h>
+
+#include <cstdarg>
+#include <cstdio>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+
+namespace {
+
+struct TypeInfo {
+ const char * name = nullptr;
+ int64_t blck_size = 0;
+ size_t type_size = 0;
+ bool quantized = false;
+};
+
+const TypeInfo * type_infos() {
+ static TypeInfo t[GGML_TYPE_COUNT];
+ static const bool init = [] {
+ auto set = [](ggml_type type, const char * name, int64_t blck, size_t size, bool quantized) {
+ t[type] = {name, blck, size, quantized};
+ };
+ set(GGML_TYPE_F32, "f32", 1, sizeof(float), false);
+ set(GGML_TYPE_F16, "f16", 1, sizeof(ggml_fp16_t), false);
+ set(GGML_TYPE_BF16, "bf16", 1, sizeof(ggml_bf16_t), false);
+ set(GGML_TYPE_F64, "f64", 1, sizeof(double), false);
+ set(GGML_TYPE_I8, "i8", 1, sizeof(int8_t), false);
+ set(GGML_TYPE_I16, "i16", 1, sizeof(int16_t), false);
+ set(GGML_TYPE_I32, "i32", 1, sizeof(int32_t), false);
+ set(GGML_TYPE_I64, "i64", 1, sizeof(int64_t), false);
+ set(GGML_TYPE_Q1_0, "q1_0", QK1_0, sizeof(block_q1_0), true);
+ set(GGML_TYPE_Q2_0, "q2_0", QK2_0, sizeof(block_q2_0), true);
+ set(GGML_TYPE_Q4_0, "q4_0", QK4_0, sizeof(block_q4_0), true);
+ set(GGML_TYPE_Q4_1, "q4_1", QK4_1, sizeof(block_q4_1), true);
+ set(GGML_TYPE_Q5_0, "q5_0", QK5_0, sizeof(block_q5_0), true);
+ set(GGML_TYPE_Q5_1, "q5_1", QK5_1, sizeof(block_q5_1), true);
+ set(GGML_TYPE_Q8_0, "q8_0", QK8_0, sizeof(block_q8_0), true);
+ set(GGML_TYPE_Q8_1, "q8_1", QK8_1, sizeof(block_q8_1), true);
+ set(GGML_TYPE_MXFP4, "mxfp4", QK_MXFP4, sizeof(block_mxfp4), true);
+ set(GGML_TYPE_NVFP4, "nvfp4", QK_NVFP4, sizeof(block_nvfp4), true);
+ set(GGML_TYPE_TQ1_0, "tq1_0", QK_K, sizeof(block_tq1_0), true);
+ set(GGML_TYPE_TQ2_0, "tq2_0", QK_K, sizeof(block_tq2_0), true);
+ set(GGML_TYPE_Q2_K, "q2_K", QK_K, sizeof(block_q2_K), true);
+ set(GGML_TYPE_Q3_K, "q3_K", QK_K, sizeof(block_q3_K), true);
+ set(GGML_TYPE_Q4_K, "q4_K", QK_K, sizeof(block_q4_K), true);
+ set(GGML_TYPE_Q5_K, "q5_K", QK_K, sizeof(block_q5_K), true);
+ set(GGML_TYPE_Q6_K, "q6_K", QK_K, sizeof(block_q6_K), true);
+ set(GGML_TYPE_Q8_K, "q8_K", QK_K, sizeof(block_q8_K), true);
+ set(GGML_TYPE_IQ2_XXS, "iq2_xxs", QK_K, sizeof(block_iq2_xxs), true);
+ set(GGML_TYPE_IQ2_XS, "iq2_xs", QK_K, sizeof(block_iq2_xs), true);
+ set(GGML_TYPE_IQ2_S, "iq2_s", QK_K, sizeof(block_iq2_s), true);
+ set(GGML_TYPE_IQ3_XXS, "iq3_xxs", QK_K, sizeof(block_iq3_xxs), true);
+ set(GGML_TYPE_IQ3_S, "iq3_s", QK_K, sizeof(block_iq3_s), true);
+ set(GGML_TYPE_IQ1_S, "iq1_s", QK_K, sizeof(block_iq1_s), true);
+ set(GGML_TYPE_IQ1_M, "iq1_m", QK_K, sizeof(block_iq1_m), true);
+ set(GGML_TYPE_IQ4_NL, "iq4_nl", QK4_NL, sizeof(block_iq4_nl), true);
+ set(GGML_TYPE_IQ4_XS, "iq4_xs", QK_K, sizeof(block_iq4_xs), true);
+ return true;
+ }();
+ (void) init;
+ return t;
+}
+
+const TypeInfo & type_info(ggml_type type) {
+ if (type < 0 || type >= GGML_TYPE_COUNT || type_infos()[type].name == nullptr) {
+ throw std::invalid_argument("unknown ggml type " + std::to_string((int) type));
+ }
+ return type_infos()[type];
+}
+
+struct EnvPool : ggml_cuda_pool {
+ explicit EnvPool(int device) : device(device) {}
+
+ void * alloc(size_t size, size_t * actual_size) override {
+ *actual_size = size;
+ if (size == 0) {
+ return nullptr;
+ }
+#ifdef GGML_USE_HIP
+ constexpr DLDeviceType device_type = kDLROCM;
+#else
+ constexpr DLDeviceType device_type = kDLCUDA;
+#endif
+ auto buf = tvm::ffi::Tensor::FromEnvAlloc(TVMFFIEnvTensorAlloc, tvm::ffi::Shape({(int64_t) size}), DLDataType{kDLUInt, 8, 1},
+ DLDevice{device_type, device});
+ void * ptr = buf.data_ptr();
+ live.emplace(ptr, std::move(buf));
+ return ptr;
+ }
+
+ void free(void * ptr, size_t) override { live.erase(ptr); }
+
+ int device;
+ std::unordered_map<void *, tvm::ffi::Tensor> live;
+};
+
+} // namespace
+
+void ggml_abort(const char * file, int line, const char * fmt, ...) {
+ char msg[1024];
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(msg, sizeof(msg), fmt, args);
+ va_end(args);
+ throw std::runtime_error(std::string("ggml: ") + file + ":" + std::to_string(line) + ": " + msg);
+}
+
+void ggml_log_internal(enum ggml_log_level level, const char * format, ...) {
+ if (level < GGML_LOG_LEVEL_WARN || level == GGML_LOG_LEVEL_CONT) {
+ return;
+ }
+ va_list args;
+ va_start(args, format);
+ vfprintf(stderr, format, args);
+ va_end(args);
+}
+
+int64_t ggml_blck_size(enum ggml_type type) { return type_info(type).blck_size; }
+
+size_t ggml_type_size(enum ggml_type type) { return type_info(type).type_size; }
+
+const char * ggml_type_name(enum ggml_type type) { return type_info(type).name; }
+
+bool ggml_is_quantized(enum ggml_type type) { return type_info(type).quantized; }
+
+size_t ggml_row_size(enum ggml_type type, int64_t ne) { return ggml_type_size(type) * ne / ggml_blck_size(type); }
+
+size_t ggml_element_size(const struct ggml_tensor * tensor) { return ggml_type_size(tensor->type); }
+
+int64_t ggml_nelements(const struct ggml_tensor * tensor) {
+ return tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3];
+}
+
+size_t ggml_nbytes(const struct ggml_tensor * tensor) {
+ for (int i = 0; i < GGML_MAX_DIMS; ++i) {
+ if (tensor->ne[i] <= 0) {
+ return 0;
+ }
+ }
+ size_t nbytes;
+ const size_t blck_size = ggml_blck_size(tensor->type);
+ if (blck_size == 1) {
+ nbytes = ggml_type_size(tensor->type);
+ for (int i = 0; i < GGML_MAX_DIMS; ++i) {
+ nbytes += (tensor->ne[i] - 1) * tensor->nb[i];
+ }
+ } else {
+ nbytes = tensor->ne[0] * tensor->nb[0] / blck_size;
+ for (int i = 1; i < GGML_MAX_DIMS; ++i) {
+ nbytes += (tensor->ne[i] - 1) * tensor->nb[i];
+ }
+ }
+ return nbytes;
+}
+
+bool ggml_is_matrix(const struct ggml_tensor * tensor) { return tensor->ne[2] == 1 && tensor->ne[3] == 1; }
+
+bool ggml_is_contiguous(const struct ggml_tensor * tensor) {
+ size_t next_nb = ggml_type_size(tensor->type);
+ if (tensor->ne[0] != ggml_blck_size(tensor->type) && tensor->nb[0] != next_nb) {
+ return false;
+ }
+ next_nb *= tensor->ne[0] / ggml_blck_size(tensor->type);
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
+ if (tensor->ne[i] != 1 && tensor->nb[i] != next_nb) {
+ return false;
+ }
+ next_nb *= tensor->ne[i];
+ }
+ return true;
+}
+
+bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor) {
+ return ggml_nbytes(tensor) == ggml_nelements(tensor) * ggml_type_size(tensor->type) / ggml_blck_size(tensor->type);
+}
+
+bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+ return t0->ne[0] == t1->ne[0] && t0->ne[1] == t1->ne[1] && t0->ne[2] == t1->ne[2] && t0->ne[3] == t1->ne[3];
+}
+
+bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
+ return t0->nb[0] == t1->nb[0] && t0->nb[1] == t1->nb[1] && t0->nb[2] == t1->nb[2] && t0->nb[3] == t1->nb[3];
+}
+
+enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage(ggml_backend_buffer_t) {
+ return GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
+}
+
+size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t, const struct ggml_tensor * tensor) {
+ return ggml_nbytes(tensor);
+}
+
+[[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {
+ throw std::runtime_error(std::string("ggml-cuda: ") + msg + " in " + func + " at " + file + ":" + std::to_string(line) + ": " + stmt);
+}
+
+int ggml_cuda_get_device() {
+ int id;
+ CUDA_CHECK(cudaGetDevice(&id));
+ return id;
+}
+
+void ggml_cuda_set_device(int device) {
+ int current;
+ CUDA_CHECK(cudaGetDevice(&current));
+ if (current != device) {
+ CUDA_CHECK(cudaSetDevice(device));
+ }
+}
+
+const ggml_cuda_device_info & ggml_cuda_info() {
+ static const ggml_cuda_device_info info = [] {
+ ggml_cuda_device_info in = {};
+ CUDA_CHECK(cudaGetDeviceCount(&in.physical_device_count));
+ GGML_ASSERT(in.physical_device_count <= GGML_CUDA_MAX_DEVICES);
+ in.device_count = in.physical_device_count;
+ for (int id = 0; id < in.device_count; ++id) {
+ cudaDeviceProp prop;
+ CUDA_CHECK(cudaGetDeviceProperties(&prop, id));
+ auto & d = in.devices[id];
+#ifdef GGML_USE_HIP
+ GGML_ASSERT(std::string(prop.gcnArchName).starts_with("gfx"));
+ d.cc = GGML_CUDA_CC_OFFSET_AMD + static_cast<int>(std::strtol(prop.gcnArchName + 3, nullptr, 16));
+#else
+ d.cc = 100 * prop.major + 10 * prop.minor;
+#endif
+ d.nsm = prop.multiProcessorCount;
+ d.smpb = prop.sharedMemPerBlock;
+ d.smpbo = prop.sharedMemPerBlockOptin;
+ d.warp_size = prop.warpSize;
+ d.total_vram = prop.totalGlobalMem;
+ d.physical_device = id;
+ d.physical_share_count = 1;
+ int coop = 0;
+ CUDA_CHECK(cudaDeviceGetAttribute(&coop, cudaDevAttrCooperativeLaunch, id));
+ d.supports_cooperative_launch = coop != 0;
+ }
+ return in;
+ }();
+ return info;
+}
+
+// The streams belong to the caller, so there is nothing to release.
+ggml_backend_cuda_context::~ggml_backend_cuda_context() {}
+
+std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(int device, int) {
+ return std::make_unique<EnvPool>(device);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuh
new file mode 100644
index 00000000..36912ce4
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuh
@@ -0,0 +1,1722 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-impl.h"
+#include "ggml-cuda.h"
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <mutex>
+
+#if defined(GGML_USE_HIP)
+#define GGML_COMMON_DECL_HIP
+#define GGML_COMMON_IMPL_HIP
+#else
+#define GGML_COMMON_DECL_CUDA
+#define GGML_COMMON_IMPL_CUDA
+#if defined(GGML_USE_MUSA)
+#define GGML_COMMON_DECL_MUSA
+#define GGML_COMMON_IMPL_MUSA
+#endif
+#endif
+#include "ggml-common.h"
+
+#include <array>
+#include <algorithm>
+#include <cassert>
+#include <cfloat>
+#include <cstdio>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#if defined(GGML_USE_HIP)
+#include "vendors/hip.h"
+#elif defined(GGML_USE_MUSA)
+#include "vendors/musa.h"
+#else
+#include "vendors/cuda.h"
+#endif // defined(GGML_USE_HIP)
+
+#define STRINGIZE_IMPL(...) #__VA_ARGS__
+#define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__)
+
+#define WARP_SIZE 32
+#define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed)
+#define CUDART_HMASK 12000 // CUDA 12.0, min. ver. for half2 -> uint mask comparisons
+
+#define GGML_CUDA_CC_PASCAL 600
+#define GGML_CUDA_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products
+#define GGML_CUDA_CC_VOLTA 700
+#define GGML_CUDA_CC_TURING 750
+#define GGML_CUDA_CC_AMPERE 800
+#define GGML_CUDA_CC_ORIN 870
+#define GGML_CUDA_CC_ADA_LOVELACE 890
+#define GGML_CUDA_CC_HOPPER 900
+// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see
+// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms
+#define GGML_CUDA_CC_BLACKWELL 1200
+#define GGML_CUDA_CC_DGX_SPARK 1210
+#define GGML_CUDA_CC_RUBIN 1300
+#define GGML_CUDA_CC_OFFSET_AMD 0x1000000
+#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000
+#define GGML_CUDA_CC_IS_NVIDIA(cc) (cc < GGML_CUDA_CC_OFFSET_MTHREADS)
+
+// AMD
+// GCN/CDNA, wave size is 64
+#define GGML_CUDA_CC_GCN4 (GGML_CUDA_CC_OFFSET_AMD + 0x803) // Tonga, Fiji, Polaris, minimum for fast fp16
+#define GGML_CUDA_CC_VEGA (GGML_CUDA_CC_OFFSET_AMD + 0x900) // Vega56/64, minimum for fp16 dual issue
+#define GGML_CUDA_CC_VEGA20 (GGML_CUDA_CC_OFFSET_AMD + 0x906) // MI50/Radeon VII, minimum for dp4a
+#define GGML_CUDA_CC_GFX909 (GGML_CUDA_CC_OFFSET_AMD + 0x909) // GCN APU
+#define GGML_CUDA_CC_GFX90C (GGML_CUDA_CC_OFFSET_AMD + 0x90c) // GCN APU
+#define GGML_CUDA_CC_CDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x908) // MI100, minimum for MFMA, acc registers
+#define GGML_CUDA_CC_CDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x90a) // MI210 (gfx90a), minimum acc register renaming
+#define GGML_CUDA_CC_CDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x942) // MI300
+#define GGML_CUDA_CC_CDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x950) // MI350X/MI355X
+
+// RDNA removes MFMA, dp4a, xnack, acc registers, wave size is 32
+#define GGML_CUDA_CC_RDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x1010) // RX 5000
+#define GGML_CUDA_CC_RDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a
+#define GGML_CUDA_CC_RDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA
+#define GGML_CUDA_CC_RDNA3_5 (GGML_CUDA_CC_OFFSET_AMD + 0x1150) // AI 370, AI Max 395 laptops.
+#define GGML_CUDA_CC_RDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000
+
+#define GGML_CUDA_CC_IS_AMD(cc) (cc >= GGML_CUDA_CC_OFFSET_AMD)
+#define GGML_CUDA_CC_IS_RDNA(cc) (cc >= GGML_CUDA_CC_RDNA1)
+#define GGML_CUDA_CC_IS_RDNA1(cc) (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2)
+#define GGML_CUDA_CC_IS_RDNA2(cc) (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3)
+#define GGML_CUDA_CC_IS_RDNA3_0(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA3_5)
+#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4)
+#define GGML_CUDA_CC_IS_RDNA3(cc) (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc))
+#define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4)
+#define GGML_CUDA_CC_IS_GCN_APU(cc) ((cc) == GGML_CUDA_CC_GFX909 || (cc) == GGML_CUDA_CC_GFX90C)
+#define GGML_CUDA_CC_IS_GCN(cc) ((cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) || GGML_CUDA_CC_IS_GCN_APU(cc))
+#define GGML_CUDA_CC_IS_CDNA(cc) (!GGML_CUDA_CC_IS_GCN_APU(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1)
+#define GGML_CUDA_CC_IS_CDNA1(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2)
+#define GGML_CUDA_CC_IS_CDNA2(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3)
+#define GGML_CUDA_CC_IS_CDNA3(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4)
+#define GGML_CUDA_CC_IS_CDNA4(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1)
+
+// Moore Threads
+#define MUSART_HMASK 40300 // MUSA rc4.3, min. ver. for half2 -> uint mask comparisons
+
+#define GGML_CUDA_CC_QY1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x210) // MTT S80, MTT S3000
+#define GGML_CUDA_CC_QY2 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x220) // MTT S4000
+#define GGML_CUDA_CC_PH1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x310) // MTT S5000
+
+#define GGML_CUDA_CC_IS_MTHREADS(cc) (cc >= GGML_CUDA_CC_OFFSET_MTHREADS && cc < GGML_CUDA_CC_OFFSET_AMD)
+#define GGML_CUDA_CC_IS_QY1(cc) (cc >= GGML_CUDA_CC_QY1 && cc < GGML_CUDA_CC_QY2)
+#define GGML_CUDA_CC_IS_QY2(cc) (cc >= GGML_CUDA_CC_QY2 && cc < GGML_CUDA_CC_PH1)
+#define GGML_CUDA_CC_IS_PH1(cc) (cc >= GGML_CUDA_CC_PH1)
+
+#if !defined(GGML_USE_HIP) && (defined(GGML_USE_MUSA) || CUDART_VERSION >= 11070)
+# define GGML_CUDA_USE_CUB
+#endif // !defined(GGML_USE_HIP) && (defined(GGML_USE_MUSA) || CUDART_VERSION >= 11070)
+
+// PDL host-side support (cudaLaunchKernelEx) requires CUDART >= 11.8.
+// However, this has been bugged in CTK < 12.3 for MSVC builds, see
+// https://github.com/ggml-org/llama.cpp/pull/22522#discussion_r3302393293
+// __CUDA_ARCH__ is undefined in host passes; GPU arch check happens in device-side code.
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && \
+ (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080))
+# define GGML_CUDA_USE_PDL
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080))
+
+static __device__ __forceinline__ void ggml_cuda_syncwarp() {
+#ifndef GGML_USE_HIP
+ __syncwarp();
+#endif // GGML_USE_HIP
+}
+
+static __device__ __forceinline__ void ggml_cuda_pdl_sync() {
+#if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER
+ cudaGridDependencySynchronize();
+#endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER
+}
+
+static __device__ __forceinline__ void ggml_cuda_pdl_lc() {
+#if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER
+ cudaTriggerProgrammaticLaunchCompletion();
+#endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER
+}
+
+#ifdef __CUDA_ARCH_LIST__
+constexpr bool ggml_cuda_has_arch_impl(int) {
+ return false;
+}
+
+template<class ... Archs>
+constexpr bool ggml_cuda_has_arch_impl(const int arch, const int first, Archs... rest) {
+ return arch == first || ggml_cuda_has_arch_impl(arch, rest...);
+}
+
+constexpr bool ggml_cuda_has_arch(const int arch) {
+ return ggml_cuda_has_arch_impl(arch, __CUDA_ARCH_LIST__);
+}
+
+constexpr int ggml_cuda_highest_compiled_arch_impl(const int /*arch*/, const int cur) {
+ if (cur == 0) {
+ return -1;
+ }
+ return cur;
+}
+
+template<class ... Archs>
+constexpr int ggml_cuda_highest_compiled_arch_impl(const int arch, const int cur, const int first, Archs... rest) {
+ if (first <= arch && first > cur) {
+ return ggml_cuda_highest_compiled_arch_impl(arch, first, rest...);
+ } else {
+ return ggml_cuda_highest_compiled_arch_impl(arch, cur, rest...);
+ }
+}
+
+constexpr int ggml_cuda_highest_compiled_arch(const int arch) {
+ return ggml_cuda_highest_compiled_arch_impl(arch, 0, __CUDA_ARCH_LIST__);
+}
+#else
+static int ggml_cuda_highest_compiled_arch(const int arch) {
+ return arch;
+}
+#endif // __CUDA_ARCH_LIST__
+
+// ---------------------------------------------------------------------------------------------------------
+
+#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses
+
+#define GGML_CUDA_MAX_STREAMS 8
+
+[[noreturn]]
+void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg);
+
+#define CUDA_CHECK_GEN(err, success, error_fn) \
+ do { \
+ auto err_ = (err); \
+ if (err_ != (success)) { \
+ ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_)); \
+ } \
+ } while (0)
+
+#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString)
+
+
+#if CUDART_VERSION >= 12000 || defined(GGML_USE_MUSA)
+ static const char * cublas_get_error_str(const cublasStatus_t err) {
+ return cublasGetStatusString(err);
+ }
+#else
+ static const char * cublas_get_error_str(const cublasStatus_t err) {
+ switch (err) {
+ case CUBLAS_STATUS_SUCCESS: return "CUBLAS_STATUS_SUCCESS";
+ case CUBLAS_STATUS_NOT_INITIALIZED: return "CUBLAS_STATUS_NOT_INITIALIZED";
+ case CUBLAS_STATUS_ALLOC_FAILED: return "CUBLAS_STATUS_ALLOC_FAILED";
+ case CUBLAS_STATUS_INVALID_VALUE: return "CUBLAS_STATUS_INVALID_VALUE";
+ case CUBLAS_STATUS_ARCH_MISMATCH: return "CUBLAS_STATUS_ARCH_MISMATCH";
+ case CUBLAS_STATUS_MAPPING_ERROR: return "CUBLAS_STATUS_MAPPING_ERROR";
+ case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED";
+ case CUBLAS_STATUS_INTERNAL_ERROR: return "CUBLAS_STATUS_INTERNAL_ERROR";
+ case CUBLAS_STATUS_NOT_SUPPORTED: return "CUBLAS_STATUS_NOT_SUPPORTED";
+ default: return "unknown error";
+ }
+ }
+#endif // CUDART_VERSION >= 12000
+
+#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str)
+
+#ifdef GGML_USE_NCCL
+#define NCCL_CHECK(err) CUDA_CHECK_GEN(err, ncclSuccess, ncclGetErrorString)
+#endif // GGML_USE_NCCL
+
+#if !defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)
+static const char * cu_get_error_str(CUresult err) {
+ const char * err_str;
+ cuGetErrorString(err, &err_str);
+ return err_str;
+}
+#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str)
+#endif
+
+#if !defined(GGML_USE_HIP)
+# define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \
+ do { \
+ static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = { false }; \
+ const int id = ggml_cuda_get_device(); \
+ if (!shared_memory_limit_raised[id]) { \
+ CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes)); \
+ shared_memory_limit_raised[id] = true; \
+ } \
+ } while (0)
+#else
+# define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \
+ do { \
+ GGML_UNUSED(nbytes); \
+ } while (0)
+#endif // !defined(GGML_USE_HIP)
+
+#if CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)
+#define GGML_CUDA_ASSUME(x) __builtin_assume(x)
+#else
+#define GGML_CUDA_ASSUME(x)
+#endif // CUDART_VERSION >= 11010
+
+#if (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM))
+#define GGML_USE_VMM
+#endif // (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM))
+
+#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL
+#define FP16_AVAILABLE
+#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL
+
+#if defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610
+#define FAST_FP16_AVAILABLE
+#endif // defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610
+
+#if defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)
+#define AMD_MFMA_AVAILABLE
+#endif // defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)
+
+#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA3))
+#define AMD_WMMA_AVAILABLE
+#endif // defined(GGML_USE_HIP) && defined(RDNA4)
+
+// The Volta instructions are in principle available on Turing or newer but they are effectively unusable:
+#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA
+#define VOLTA_MMA_AVAILABLE
+#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA
+
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING
+#define TURING_MMA_AVAILABLE
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING
+
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#define AMPERE_MMA_AVAILABLE
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+
+// FreeToken: the native fp4 MMA needs an sm_120a/sm_120f build, so it is off unless GGML_CUDA_FP4_MMA is defined; FP4 weights then take Q8_1 activations
+#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN && defined(GGML_CUDA_FP4_MMA)
+# define BLACKWELL_MMA_AVAILABLE
+#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL
+
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#define CP_ASYNC_AVAILABLE
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+
+#if !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220)
+#define FLASH_ATTN_AVAILABLE
+#endif // !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220)
+
+static bool fp16_available(const int cc) {
+ return ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_PASCAL ||
+ (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1);
+}
+
+static bool fast_fp16_available(const int cc) {
+ return GGML_CUDA_CC_IS_AMD(cc) ||
+ (GGML_CUDA_CC_IS_NVIDIA(cc) && fp16_available(cc) && ggml_cuda_highest_compiled_arch(cc) != 610) ||
+ (GGML_CUDA_CC_IS_MTHREADS(cc) && fp16_available(cc));
+}
+
+// To be used for feature selection of external libraries, e.g. cuBLAS.
+static bool fast_fp16_hardware_available(const int cc) {
+ return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_PASCAL && cc != 610) || GGML_CUDA_CC_IS_AMD(cc) ||
+ (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);
+}
+
+// To be used for feature selection of external libraries, e.g. cuBLAS.
+static bool fp16_mma_hardware_available(const int cc) {
+ return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_VOLTA) ||
+ GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc) ||
+ (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);
+}
+
+// To be used for feature selection of external libraries, e.g. cuBLAS.
+static bool fast_bf16_hardware_available(const int cc) {
+ return (GGML_CUDA_CC_IS_AMD(cc) && (cc >= GGML_CUDA_CC_RDNA3 || GGML_CUDA_CC_IS_CDNA(cc)))
+ || (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE);
+}
+
+static bool bf16_mma_hardware_available(const int cc) {
+ return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE) ||
+ GGML_CUDA_CC_IS_CDNA(cc) || cc >= GGML_CUDA_CC_RDNA3 ||
+ (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1);
+}
+
+static bool fp32_mma_hardware_available(const int cc) {
+ return GGML_CUDA_CC_IS_CDNA(cc);
+}
+
+static bool amd_mfma_available(const int cc) {
+#if !defined(GGML_HIP_NO_MMQ_MFMA)
+ return GGML_CUDA_CC_IS_CDNA(cc);
+#else
+ return false;
+#endif //!defined(GGML_HIP_NO_MMQ_MFMA)
+}
+
+static bool amd_wmma_available(const int cc) {
+ return (GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_RDNA3(cc));
+}
+
+static bool volta_mma_available(const int cc) {
+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_VOLTA;
+}
+
+static bool turing_mma_available(const int cc) {
+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_TURING;
+}
+
+static bool ampere_mma_available(const int cc) {
+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;
+}
+
+static bool cp_async_available(const int cc) {
+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;
+}
+
+static bool blackwell_mma_available(const int cc) {
+#ifdef GGML_CUDA_FP4_MMA
+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL &&
+ ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;
+#else
+ GGML_UNUSED(cc);
+ return false;
+#endif // GGML_CUDA_FP4_MMA
+}
+
+// Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes.
+static bool ggml_cuda_is_aligned(const ggml_tensor * tensor, const size_t alignment) {
+ GGML_ASSERT(tensor != nullptr);
+ return (reinterpret_cast<uintptr_t>(tensor->data) % alignment) == 0 &&
+ tensor->nb[1] % alignment == 0 &&
+ tensor->nb[2] % alignment == 0 &&
+ tensor->nb[3] % alignment == 0;
+}
+
+static constexpr __device__ int ggml_cuda_get_physical_warp_size() {
+#if defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))
+ return 64;
+#else
+ return 32;
+#endif // defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))
+}
+
+// Maximum number of bytes that can be copied in a single instruction.
+static constexpr __device__ int ggml_cuda_get_max_cpy_bytes() {
+#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
+ return 16;
+#else
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
+ return 16;
+#else
+ return 8;
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
+#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
+}
+
+
+[[noreturn]]
+static __device__ void no_device_code(
+ const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {
+
+#if defined(GGML_USE_HIP)
+ printf("%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\n",
+ file_name, line, function_name, arch);
+ GGML_UNUSED(arch_list);
+#else
+ printf("%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\n",
+ file_name, line, function_name, arch, arch_list);
+#endif // defined(GGML_USE_HIP)
+ __trap();
+
+ GGML_UNUSED(no_device_code); // suppress unused function warning
+}
+
+#ifdef __CUDA_ARCH__
+#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__))
+#else
+#define NO_DEVICE_CODE //GGML_ABORT("NO_DEVICE_CODE not valid in host code.")
+#endif // __CUDA_ARCH__
+
+// The compiler is always able to unroll loops if they contain continue expressions.
+// In such cases loop unrolling can still be achieved via recursion:
+template <int n>
+struct ggml_cuda_unroll {
+ template <typename Func, typename... Args>
+ __device__ void operator()(const Func & f, Args... args) const {
+ f(n - 1, args...);
+ ggml_cuda_unroll<n - 1>{}(f, args...);
+ }
+};
+
+template <>
+struct ggml_cuda_unroll<1> {
+ template <typename Func, typename... Args>
+ __device__ void operator()(const Func & f, Args... args) const {
+ f(0, args...);
+ }
+};
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ int warp_reduce_sum(int x) {
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+ return __reduce_add_sync(0xffffffff, x);
+#else
+#pragma unroll
+ for (int offset = width/2; offset > 0; offset >>= 1) {
+ x += __shfl_xor_sync(0xffffffff, x, offset, width);
+ }
+ return x;
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ float warp_reduce_sum(float x) {
+#pragma unroll
+ for (int offset = width/2; offset > 0; offset >>= 1) {
+ x += __shfl_xor_sync(0xffffffff, x, offset, width);
+ }
+ return x;
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {
+#pragma unroll
+ for (int offset = width/2; offset > 0; offset >>= 1) {
+ a.x += __shfl_xor_sync(0xffffffff, a.x, offset, width);
+ a.y += __shfl_xor_sync(0xffffffff, a.y, offset, width);
+ }
+ return a;
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {
+#ifdef FP16_AVAILABLE
+#pragma unroll
+ for (int offset = width/2; offset > 0; offset >>= 1) {
+ a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, offset, width));
+ }
+ return a;
+
+#else
+ NO_DEVICE_CODE;
+ return a;
+#endif // FP16_AVAILABLE
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ int warp_reduce_all(int x) {
+ if (width == ggml_cuda_get_physical_warp_size()) {
+ return __all_sync(0xffffffff, x);
+ } else {
+#pragma unroll
+ for (int offset = width/2; offset > 0; offset >>= 1) {
+ x = __shfl_xor_sync(0xffffffff, x, offset, width) && x;
+ }
+ return x;
+ }
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ int warp_reduce_any(int x) {
+ if (width == ggml_cuda_get_physical_warp_size()) {
+ return __any_sync(0xffffffff, x);
+ } else {
+#pragma unroll
+ for (int offset = width/2; offset > 0; offset >>= 1) {
+ x = __shfl_xor_sync(0xffffffff, x, offset, width) || x;
+ }
+ return x;
+ }
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ float warp_reduce_max(float x) {
+#pragma unroll
+ for (int offset = width/2; offset > 0; offset >>= 1) {
+ x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, offset, width));
+ }
+ return x;
+}
+
+template<typename T, int width = WARP_SIZE>
+static __device__ __forceinline__ T warp_prefix_inclusive_sum(T x) {
+ const int lane_id = threadIdx.x % width;
+#pragma unroll
+ for (int offset = 1; offset < width; offset <<= 1) {
+ const T t = __shfl_up_sync(0xffffffff, x, offset, width);
+ if (lane_id >= offset) {
+ x += t;
+ }
+ }
+ return x;
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ float2 warp_prefix_inclusive_sum(float2 a) {
+ const int lane_id = threadIdx.x % width;
+#pragma unroll
+ for (int offset = 1; offset < width; offset <<= 1) {
+ const float t_x = __shfl_up_sync(0xffffffff, a.x, offset, width);
+ const float t_y = __shfl_up_sync(0xffffffff, a.y, offset, width);
+ if (lane_id >= offset) {
+ a.x += t_x;
+ a.y += t_y;
+ }
+ }
+ return a;
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ half2 warp_prefix_inclusive_sum(half2 a) {
+#ifdef FP16_AVAILABLE
+ const int lane_id = threadIdx.x % width;
+#pragma unroll
+ for (int offset = 1; offset < width; offset <<= 1) {
+ const half2 t = __shfl_up_sync(0xffffffff, a, offset, width);
+ if (lane_id >= offset) {
+ a = __hadd2(a, t);
+ }
+ }
+ return a;
+
+#else
+ NO_DEVICE_CODE;
+ return a;
+#endif // FP16_AVAILABLE
+}
+
+enum class block_reduce_method {
+ MAX,
+ SUM,
+};
+
+template<block_reduce_method method_t, typename T>
+struct block_reduce_policy;
+
+template <typename T, typename... Ts>
+inline constexpr bool is_any = (std::is_same_v<T, Ts> || ...);
+
+template<typename...>
+inline constexpr bool ggml_cuda_dependent_false_v = false;
+
+template <typename T> struct block_reduce_policy<block_reduce_method::SUM, T> {
+ static __device__ T reduce(T val) {
+ if constexpr(is_any<T, float, float2, half2, int>) {
+ return warp_reduce_sum(val);
+ } else {
+ static_assert(ggml_cuda_dependent_false_v<T>, "Unsupported type for block reduce sum");
+ }
+ }
+
+ static __device__ T sentinel() {
+ if constexpr (std::is_same_v<T, float>) {
+ return 0.0f;
+ } else if constexpr (std::is_same_v<T, float2>) {
+ return make_float2(0.0f, 0.0f);
+ } else if constexpr (std::is_same_v<T, half2>) {
+ return make_half2(0.0f, 0.0f);
+ } else if constexpr (std::is_same_v<T, int>) {
+ return 0;
+ } else {
+ static_assert(ggml_cuda_dependent_false_v<T>, "Unsupported type for block reduce sum");
+ }
+ }
+};
+
+template <typename T> struct block_reduce_policy<block_reduce_method::MAX, T> {
+ static __device__ T reduce(T val) {
+ if constexpr (is_any<T, float, half2>) {
+ return warp_reduce_max(val);
+ } else {
+ static_assert(ggml_cuda_dependent_false_v<T>, "Unsupported type for block reduce max");
+ }
+ }
+
+ static __device__ T sentinel() {
+ if constexpr (std::is_same_v<T, float>) {
+ return -INFINITY;
+ } else if constexpr (std::is_same_v<T, half2>) {
+ return make_half2(-INFINITY, -INFINITY);
+ } else {
+ static_assert(ggml_cuda_dependent_false_v<T>, "Unsupported type for block reduce max");
+ }
+ }
+};
+
+template <block_reduce_method reduce_method_t, const unsigned int block_size_template = 0, typename T>
+static __device__ T block_reduce(T val, [[maybe_unused]] T * shared_vals) {
+ // for multi-warp reductions, callers must not reuse shared_vals until all reads from this invocation have completed
+ val = block_reduce_policy<reduce_method_t, T>::reduce(val);
+ const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template;
+ if (block_size > WARP_SIZE) {
+ assert((block_size <= 1024) && (block_size % WARP_SIZE) == 0);
+ const int warp_id = threadIdx.x / WARP_SIZE;
+ const int lane_id = threadIdx.x % WARP_SIZE;
+ if (lane_id == 0) {
+ shared_vals[warp_id] = val;
+ }
+ __syncthreads();
+ val = block_reduce_policy<reduce_method_t, T>::sentinel();
+ if (lane_id < (static_cast<int>(block_size) / WARP_SIZE)) {
+ val = shared_vals[lane_id];
+ }
+ return block_reduce_policy<reduce_method_t, T>::reduce(val);
+ }
+
+ return val;
+}
+
+static __device__ __forceinline__ half ggml_cuda_hmax(const half a, const half b) {
+#ifdef FP16_AVAILABLE
+
+#if !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX
+ return __float2half(fmaxf(__half2float(a), __half2float(b)));
+#else
+ return __hmax(a, b);
+#endif // !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX
+
+#else
+ NO_DEVICE_CODE;
+ GGML_UNUSED(b);
+ return a;
+#endif // FP16_AVAILABLE
+}
+
+static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const half2 b) {
+#if defined(GGML_USE_HIP)
+ return half2(__hmax(a.x, b.x), __hmax(a.y, b.y));
+#elif CUDART_VERSION >= CUDART_HMAX
+ return __hmax2(a, b);
+#else
+ half2 ret;
+ reinterpret_cast<half&>(ret.x) = __float2half(fmaxf( __low2float(a), __low2float(b)));
+ reinterpret_cast<half&>(ret.y) = __float2half(fmaxf(__high2float(a), __high2float(b)));
+ return ret;
+#endif
+}
+
+template<int width = WARP_SIZE>
+static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {
+#pragma unroll
+ for (int offset = width/2; offset > 0; offset >>= 1) {
+ x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, offset, width));
+ }
+ return x;
+}
+
+#if (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || \
+ (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK)
+static __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) {
+ const uint32_t mask_low = 0x0000FFFF * (float( __low2half(a)) > float( __low2half(b)));
+ const uint32_t mask_high = 0xFFFF0000 * (float(__high2half(a)) > float(__high2half(b)));
+ return mask_low | mask_high;
+}
+#endif // (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK)
+
+static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, int c) {
+#if defined(GGML_USE_HIP)
+#if defined(CDNA) || defined(RDNA2) || defined(__gfx906__)
+ c = __builtin_amdgcn_sdot4(a, b, c, false);
+#elif defined(RDNA3) || defined(RDNA4)
+ c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);
+#elif defined(RDNA1) || defined(__gfx900__)
+ int tmp1;
+ int tmp2;
+ asm("\n \
+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \n \
+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \n \
+ v_add3_u32 %0, %1, %2, %0 \n \
+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \n \
+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \n \
+ v_add3_u32 %0, %1, %2, %0 \n \
+ "
+ : "+v"(c), "=&v"(tmp1), "=&v"(tmp2)
+ : "v"(a), "v"(b)
+ );
+#else
+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
+ c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];
+#endif
+ return c;
+
+#else // defined(GGML_USE_HIP)
+
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)
+ return __dp4a(a, b, c);
+#else // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)
+ const int8_t * a8 = (const int8_t *) &a;
+ const int8_t * b8 = (const int8_t *) &b;
+ return c + a8[0]*b8[0] + a8[1]*b8[1] + a8[2]*b8[2] + a8[3]*b8[3];
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)
+
+#endif // defined(GGML_USE_HIP)
+}
+
+static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float v, const float u) {
+ acc += v*u;
+}
+
+static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float2 v, const float2 u) {
+ acc += v.x*u.x;
+ acc += v.y*u.y;
+}
+
+#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
+#define V_DOT2_F32_F16_AVAILABLE
+#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
+
+static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const half2 v, const half2 u) {
+#ifdef V_DOT2_F32_F16_AVAILABLE
+ asm volatile("v_dot2_f32_f16 %0, %1, %2, %0" : "+v"(acc) : "v"(v), "v"(u));
+#else
+#ifdef FAST_FP16_AVAILABLE
+ const float2 tmp = __half22float2(v*u);
+ acc += tmp.x + tmp.y;
+#else
+ const float2 tmpv = __half22float2(v);
+ const float2 tmpu = __half22float2(u);
+ acc += tmpv.x * tmpu.x;
+ acc += tmpv.y * tmpu.y;
+#endif // FAST_FP16_AVAILABLE
+#endif // V_DOT2_F32_F16_AVAILABLE
+}
+
+static __device__ __forceinline__ void ggml_cuda_mad(half2 & acc, const half2 v, const half2 u) {
+#ifdef FAST_FP16_AVAILABLE
+ acc += v*u;
+#else
+ const float2 tmpv = __half22float2(v);
+ const float2 tmpu = __half22float2(u);
+ float2 tmpacc = __half22float2(acc);
+ tmpacc.x += tmpv.x * tmpu.x;
+ tmpacc.y += tmpv.y * tmpu.y;
+ acc = make_half2(tmpacc.x, tmpacc.y);
+#endif // FAST_FP16_AVAILABLE
+}
+
+// Aligned memory transfers of 8/16 bytes can be faster than 2 transfers with 4 bytes, especially on AMD.
+// Important: do not use this function if dst and src both point at registers.
+// Due to the strict aliasing rule the compiler can do incorrect optimizations if src and dst have different types.
+// The function is intended for copies between registers and SRAM/VRAM to make the compiler emit the right instructions.
+// If dst and src point at different address spaces then they are guaranteed to not be aliased.
+template <int nbytes, int alignment = 0>
+static __device__ __forceinline__ void ggml_cuda_memcpy_1(void * __restrict__ dst, const void * __restrict__ src) {
+ static_assert(
+ nbytes <= ggml_cuda_get_max_cpy_bytes() || alignment == 0,
+ "You are misusing the alignment parameter for ggml_cuda_memcpy_1. "
+ "The intent is for the parameter is only as a workaround if either one of the pointers is not properly aligned. "
+ "If you use it to do more bytes per copy than ggml_cuda_max_cpy_bytes() the reads and writes may not be coalesced. "
+ "Call ggml_cuda_memcpy_1 in a loop instead.");
+ if constexpr (alignment != 0) {
+ static_assert(nbytes % alignment == 0, "bad alignment");
+ }
+ constexpr int nb_per_cpy = alignment == 0 ? nbytes : alignment;
+
+#pragma unroll
+ for (int i = 0; i < nbytes/nb_per_cpy; ++i) {
+ if constexpr (nb_per_cpy == 1) {
+ ((char *) dst)[i] = ((const char *) src)[i];
+ } else if constexpr (nb_per_cpy == 2) {
+ ((short *) dst)[i] = ((const short *) src)[i];
+ } else if constexpr (nb_per_cpy == 4) {
+ ((int *) dst)[i] = ((const int *) src)[i];
+ } else if constexpr (nb_per_cpy == 8) {
+ ((int2 *) dst)[i] = ((const int2 *) src)[i];
+ } else if constexpr (nb_per_cpy == 16) {
+ ((int4 *) dst)[i] = ((const int4 *) src)[i];
+ } else {
+ static_assert(nbytes == 0 && nbytes == -1, "bad nbytes");
+ }
+ }
+}
+
+static __device__ __forceinline__ float ggml_cuda_e8m0_to_fp32(uint8_t x) {
+#if CUDART_VERSION >= 12080
+ const nv_bfloat16 e = __nv_cvt_e8m0_to_bf16raw(x);
+ return (float) e;
+#else
+ uint32_t bits;
+ if (x == 0) {
+ bits = 0x00400000;
+ } else {
+ bits = (uint32_t) x << 23;
+ }
+
+ float result;
+ memcpy(&result, &bits, sizeof(float));
+ return result;
+#endif // CUDART_VERSION >= 12050
+}
+
+static __device__ __forceinline__ float ggml_cuda_ue4m3_to_fp32(uint8_t x) {
+#if defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000
+ // ROCm does not support fp8 in software on devices with fp8 hardware,
+ // but CDNA3 supports only e4m3_fnuz (no inf).
+ const uint32_t bits = x * (x != 0x7F && x != 0xFF); // Convert NaN to 0.0f to match CPU implementation.
+ const __hip_fp8_e4m3_fnuz xf = *reinterpret_cast<const __hip_fp8_e4m3_fnuz *>(&bits);
+ return static_cast<float>(xf) / 2;
+#else
+#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP)
+ const uint32_t bits = x * (x != 0x7F && x != 0xFF); // Convert NaN to 0.0f to match CPU implementation.
+ const __nv_fp8_e4m3 xf = *reinterpret_cast<const __nv_fp8_e4m3 *>(&bits);
+ return static_cast<float>(xf) / 2;
+#else
+ if (x == 0 || (x == 0x7F && x != 0xFF)) { // Convert NaN to 0.0f
+ return 0.0f;
+ }
+ const int exp = (x >> 3) & 0xF;
+ const int man = x & 0x7;
+ float raw;
+ if (exp == 0) {
+ raw = ldexpf((float) man, -9);
+ } else {
+ raw = ldexpf(1.0f + (float) man / 8.0f, exp - 7);
+ }
+ return static_cast<float>(raw / 2);
+#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP)
+#endif // defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000
+}
+
+static __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_ue4m3(float x) {
+#if defined(BLACKWELL_MMA_AVAILABLE) // This is used for NVFP4 subblock scale quantizations only
+ if (!(x > 0.0f)) {
+ return 0;
+ }
+ const __nv_fp8_e4m3 xf(x);
+ return xf.__x;
+#else
+ GGML_UNUSED(x);
+ NO_DEVICE_CODE; // Used only for NVFP4 Scales for Activations, only for Blackwell
+#endif // defined(BLACKWELL_MMA_AVAILABLE)
+}
+
+__device__ __forceinline__ uint8_t ggml_cuda_float_to_fp4_e2m1(float x, float e) {
+ const uint8_t sign_bit = (x < 0.0f) << 3;
+ float ax = fabsf(x) * e;
+
+ // Positive LUT
+ static constexpr float pos_lut[8] = { 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f };
+
+ int best_i = 0;
+ float best_err = fabsf(ax - pos_lut[0]);
+
+#pragma unroll
+ for (int i = 1; i < 8; ++i) {
+ const float err = fabsf(ax - pos_lut[i]);
+ if (err < best_err) {
+ best_err = err;
+ best_i = i;
+ }
+ }
+
+ return static_cast<uint8_t>(best_i | sign_bit);
+}
+
+// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.
+// Precompute mp (m' in the paper) and L such that division
+// can be computed using a multiply (high 32b of 64b result)
+// and a shift:
+//
+// n/d = (mulhi(n, mp) + n) >> L;
+static const uint3 init_fastdiv_values(uint64_t d_64) {
+ GGML_ASSERT(d_64 != 0);
+ GGML_ASSERT(d_64 <= std::numeric_limits<uint32_t>::max());
+
+ uint32_t d = (uint32_t)d_64;
+
+ // compute L = ceil(log2(d));
+ uint32_t L = 0;
+ while (L < 32 && (uint32_t{ 1 } << L) < d) {
+ L++;
+ }
+
+ uint32_t mp = (uint32_t) ((uint64_t{ 1 } << 32) * ((uint64_t{ 1 } << L) - d) / d + 1);
+ // pack divisor as well to reduce error surface
+ return make_uint3(mp, L, d);
+}
+
+static __device__ __forceinline__ uint32_t fastdiv(uint32_t n, const uint3 fastdiv_values) {
+ // expects fastdiv_values to contain <mp, L, divisor> in <x, y, z>
+ // fastdiv_values.z is unused and optimized away by the compiler.
+ // Compute high 32 bits of n * mp
+ const uint32_t hi = __umulhi(n, fastdiv_values.x);
+ // add n, apply bit shift
+ return (hi + n) >> fastdiv_values.y;
+}
+
+static __device__ __forceinline__ uint32_t fastmodulo(uint32_t n, const uint3 fastdiv_values) {
+ // expects fastdiv_values to contain <mp, L, divisor> in <x, y, z> (see init_fastdiv_values)
+ return n - fastdiv(n, fastdiv_values) * fastdiv_values.z;
+}
+
+// Calculate both division and modulo at once, returns <n/divisor, n%divisor>
+static __device__ __forceinline__ uint2 fast_div_modulo(uint32_t n, const uint3 fastdiv_values) {
+ // expects fastdiv_values to contain <mp, L, divisor> in <x, y, z> (see init_fastdiv_values)
+ const uint32_t div_val = fastdiv(n, fastdiv_values);
+ const uint32_t mod_val = n - div_val * fastdiv_values.z;
+ return make_uint2(div_val, mod_val);
+}
+
+typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, float2 & v);
+
+template<typename dst_t>
+using dequantize_kq_t = void (*)(const void * vx, const int64_t ib, dst_t * y, const int tid);
+
+static __device__ __forceinline__ float get_alibi_slope(
+ const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1
+) {
+ if (max_bias <= 0.0f) {
+ return 1.0f;
+ }
+ const float base = h < n_head_log2 ? m0 : m1;
+ const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
+
+ return powf(base, exph);
+}
+
+template <ggml_type type>
+struct ggml_cuda_type_traits;
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_F16> {
+ static constexpr int qk = 1;
+ static constexpr int qr = 1;
+ static constexpr int bs = sizeof(ggml_half);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q1_0> {
+ static constexpr int qk = QK1_0;
+ static constexpr int qr = QR1_0;
+ static constexpr int qi = QI1_0;
+ static constexpr int bs = sizeof(block_q1_0);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q2_0> {
+ static constexpr int qk = QK2_0;
+ static constexpr int qr = QR2_0;
+ static constexpr int qi = QI2_0;
+ static constexpr int bs = sizeof(block_q2_0);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q4_0> {
+ static constexpr int qk = QK4_0;
+ static constexpr int qr = QR4_0;
+ static constexpr int qi = QI4_0;
+ static constexpr int bs = sizeof(block_q4_0);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q4_1> {
+ static constexpr int qk = QK4_1;
+ static constexpr int qr = QR4_1;
+ static constexpr int qi = QI4_1;
+ static constexpr int bs = sizeof(block_q4_1);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q5_0> {
+ static constexpr int qk = QK5_0;
+ static constexpr int qr = QR5_0;
+ static constexpr int qi = QI5_0;
+ static constexpr int bs = sizeof(block_q5_0);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q5_1> {
+ static constexpr int qk = QK5_1;
+ static constexpr int qr = QR5_1;
+ static constexpr int qi = QI5_1;
+ static constexpr int bs = sizeof(block_q5_1);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q8_0> {
+ static constexpr int qk = QK8_0;
+ static constexpr int qr = QR8_0;
+ static constexpr int qi = QI8_0;
+ static constexpr int bs = sizeof(block_q8_0);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_MXFP4> {
+ static constexpr int qk = QK_MXFP4;
+ static constexpr int qr = QR_MXFP4;
+ static constexpr int qi = QI_MXFP4;
+ static constexpr int bs = sizeof(block_mxfp4);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_NVFP4> {
+ static constexpr int qk = QK_NVFP4;
+ static constexpr int qr = QR_NVFP4;
+ static constexpr int qi = QI_NVFP4;
+ static constexpr int bs = sizeof(block_nvfp4);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q2_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_K;
+ static constexpr int qi = QI2_K;
+ static constexpr int bs = sizeof(block_q2_K);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q3_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR3_K;
+ static constexpr int qi = QI3_K;
+ static constexpr int bs = sizeof(block_q3_K);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q4_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR4_K;
+ static constexpr int qi = QI4_K;
+ static constexpr int bs = sizeof(block_q4_K);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q5_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR5_K;
+ static constexpr int qi = QI5_K;
+ static constexpr int bs = sizeof(block_q5_K);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_Q6_K> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR6_K;
+ static constexpr int qi = QI6_K;
+ static constexpr int bs = sizeof(block_q6_K);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XXS> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_XXS;
+ static constexpr int qi = QI2_XXS;
+ static constexpr int bs = sizeof(block_iq2_xxs);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XS> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_XS;
+ static constexpr int qi = QI2_XS;
+ static constexpr int bs = sizeof(block_iq2_xs);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_S> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_S;
+ static constexpr int qi = QI2_S;
+ static constexpr int bs = sizeof(block_iq2_s);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ3_XXS> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR3_XXS;
+ static constexpr int qi = QI3_XXS;
+ static constexpr int bs = sizeof(block_iq3_xxs);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ1_S> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR1_S;
+ static constexpr int qi = QI1_S;
+ static constexpr int bs = sizeof(block_iq1_s);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ1_M> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR1_M;
+ static constexpr int qi = QI1_M;
+ static constexpr int bs = sizeof(block_iq1_m);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ4_NL> {
+ static constexpr int qk = QK4_NL;
+ static constexpr int qr = QR4_NL;
+ static constexpr int qi = QI4_NL;
+ static constexpr int bs = sizeof(block_iq4_nl);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ4_XS> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR4_XS;
+ static constexpr int qi = QI4_XS;
+ static constexpr int bs = sizeof(block_iq4_xs);
+};
+
+template<>
+struct ggml_cuda_type_traits<GGML_TYPE_IQ3_S> {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR3_S;
+ static constexpr int qi = QI3_S;
+ static constexpr int bs = sizeof(block_iq3_s);
+};
+
+//////////////////////
+
+struct ggml_cuda_device_info {
+ int device_count; // number of (possibly virtual) devices exposed to the rest of ggml
+ int physical_device_count; // number of physical CUDA devices actually present
+
+ struct cuda_device_info {
+ int cc; // compute capability
+ int nsm; // number of streaming multiprocessors
+ size_t smpb; // max. shared memory per block
+ size_t smpbo; // max. shared memory per block (with opt-in)
+ bool integrated; // Device is integrated as opposed to discrete
+ bool vmm; // virtual memory support
+ size_t vmm_granularity; // granularity of virtual memory
+ size_t total_vram;
+ int warp_size; // Number of threads in a dispatch
+ bool supports_cooperative_launch; // whether cooperative launch is supported
+ int physical_device; // backing physical CUDA device for this (virtual) device
+ int physical_share_count; // number of (virtual) devices sharing this device's physical GPU
+ int virtual_index; // index of this (virtual) device among those sharing its physical GPU
+ };
+
+ cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {};
+
+ std::array<float, GGML_CUDA_MAX_DEVICES> default_tensor_split = {};
+};
+
+const ggml_cuda_device_info & ggml_cuda_info();
+
+void ggml_cuda_set_device(int device);
+int ggml_cuda_get_device();
+
+struct ggml_cuda_pool {
+ virtual ~ggml_cuda_pool() = default;
+
+ virtual void * alloc(size_t size, size_t * actual_size) = 0;
+ virtual void free(void * ptr, size_t size) = 0;
+};
+
+template<typename T>
+struct ggml_cuda_pool_alloc {
+ ggml_cuda_pool * pool = nullptr;
+ T * ptr = nullptr;
+ size_t actual_size = 0;
+
+ ggml_cuda_pool_alloc() = default;
+
+ explicit ggml_cuda_pool_alloc(ggml_cuda_pool & pool) : pool(&pool) {
+ }
+
+ ggml_cuda_pool_alloc(ggml_cuda_pool & pool, size_t size) : pool(&pool) {
+ alloc(size);
+ }
+
+ ~ggml_cuda_pool_alloc() {
+ if (ptr != nullptr) {
+ pool->free(ptr, actual_size);
+ }
+ }
+
+ // size is in number of elements
+ T * alloc(size_t size) {
+ GGML_ASSERT(pool != nullptr);
+ GGML_ASSERT(ptr == nullptr);
+ ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size);
+ return ptr;
+ }
+
+ T * alloc(ggml_cuda_pool & pool, size_t size) {
+ this->pool = &pool;
+ return alloc(size);
+ }
+
+ T * get() {
+ return ptr;
+ }
+
+ ggml_cuda_pool_alloc(const ggml_cuda_pool_alloc &) = delete;
+ ggml_cuda_pool_alloc(ggml_cuda_pool_alloc &&) = delete;
+ ggml_cuda_pool_alloc& operator=(const ggml_cuda_pool_alloc &) = delete;
+ ggml_cuda_pool_alloc& operator=(ggml_cuda_pool_alloc &&) = delete;
+};
+
+
+// backend interface
+
+struct ggml_tensor_extra_gpu {
+ void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors
+ cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs
+};
+
+
+#if (defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)) || defined(GGML_MUSA_GRAPHS)
+#define USE_CUDA_GRAPH
+#endif
+
+struct ggml_cuda_graph {
+#ifdef USE_CUDA_GRAPH
+ ~ggml_cuda_graph() {
+ if (instance != nullptr) {
+ CUDA_CHECK(cudaGraphExecDestroy(instance));
+ }
+ if (graph != nullptr) {
+ CUDA_CHECK(cudaGraphDestroy(graph));
+ }
+ }
+ cudaGraph_t graph = nullptr;
+ cudaGraphExec_t instance = nullptr;
+ size_t num_nodes = 0;
+ std::vector<cudaGraphNode_t> nodes;
+ bool disable_due_to_gpu_arch = false;
+ bool warmup_complete = false;
+ uint64_t uid = 0;
+ int64_t last_used_time = 0;
+ struct node_properties {
+ ggml_tensor node;
+ void * node_src_data_ptrs[GGML_MAX_SRC];
+ int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS];
+ size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS];
+ };
+ std::vector<node_properties> node_props;
+
+ bool is_enabled() const {
+ static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr);
+ return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env);
+ }
+#endif
+};
+
+struct ggml_cuda_concurrent_event {
+ std::vector<cudaEvent_t> join_events;
+ cudaEvent_t fork_event = nullptr;
+
+ int n_streams = 0;
+ std::unordered_map<const ggml_tensor *, int> stream_mapping;
+
+ // Original order of nodes in this concurrent region (before interleaving)
+ // Used to restore grouping for fusion within streams
+ std::vector<const ggml_tensor *> original_order;
+
+ const ggml_tensor * join_node;
+
+ ggml_cuda_concurrent_event() = default;
+
+ ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete;
+ ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete;
+
+ explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) {
+ join_events.resize(n_streams);
+
+ for (size_t i = 0; i < join_events.size(); ++i) {
+ CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming));
+ }
+
+ CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming));
+ }
+
+ ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept
+ : join_events(std::move(other.join_events))
+ , fork_event(other.fork_event)
+ , n_streams(other.n_streams)
+ , stream_mapping(std::move(other.stream_mapping))
+ , original_order(std::move(other.original_order))
+ , join_node(other.join_node) {
+ other.fork_event = nullptr;
+ }
+
+ // 1. check if any branches write to overlapping memory ranges (except the join node)
+ // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event
+ // we assume all nodes have the same buffer
+ bool is_valid() const {
+ std::vector<std::vector<std::pair<int64_t, int64_t>>> write_ranges;
+ write_ranges.resize(n_streams);
+
+ // get join_node's memory range to exclude from overlap checking.
+ // multiple nodes can use join_node's buffer; we synchronize on the join node.
+ const ggml_tensor * join_t = join_node->view_src ? join_node->view_src : join_node;
+ const int64_t join_start = (int64_t) join_t->data;
+ const int64_t join_end = join_start + ggml_nbytes(join_t);
+
+ for (const auto & [tensor, stream] : stream_mapping) {
+ const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor;
+ const int64_t t_start = (int64_t) t->data;
+ const int64_t t_end = t_start + ggml_nbytes(t);
+
+ // skip tensors that overlap with join_node's buffer.
+ if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) {
+ continue;
+ }
+
+ // concurrent streams begin from 1
+ write_ranges[stream - 1].emplace_back(t_start, t_end);
+ }
+
+ for (int i = 0; i < n_streams; ++i) {
+ // sorts first by start then by end of write range
+ std::sort(write_ranges[i].begin(), write_ranges[i].end());
+ }
+
+ bool writes_overlap = false;
+ bool dependent_srcs = false;
+ for (const auto & [tensor, stream] : stream_mapping) {
+ const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor;
+ const int64_t t_start = (int64_t) t->data;
+ const int64_t t_end = t_start + ggml_nbytes(t);
+
+ // skip tensors that overlap with join_node's buffer
+ if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) {
+ continue;
+ }
+
+ // check if this buffer's write data overlaps with another stream's
+ std::pair<int64_t, int64_t> data_range = std::make_pair(t_start, t_end);
+ for (int i = 0; i < n_streams; ++i) {
+ if (i == stream - 1) {
+ continue;
+ }
+ auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range);
+
+ if (it != write_ranges[i].end()) {
+ const std::pair<int64_t, int64_t> & other = *it;
+
+ // std::lower_bound returns the first element where other >= data_range (lexicographically).
+ // This guarantees other.first >= data_range.first.
+ // Therefore, overlap occurs iff other.first < data_range.second
+ // (i.e., the other range starts before this range ends).
+ if (other.first < data_range.second) {
+ GGML_LOG_DEBUG("Writes overlap for %s", tensor->name);
+ writes_overlap = true;
+ break;
+ }
+ }
+ }
+
+ //check if all srcs are either in branch or don't have a branch
+ for (int i = 0; i < GGML_MAX_SRC; ++i) {
+ if (!tensor->src[i]) {
+ continue;
+ }
+
+ auto it = stream_mapping.find(tensor->src[i]);
+
+ if (it == stream_mapping.end()) {
+ continue;
+ }
+
+ if (it->second != stream) {
+ dependent_srcs = true;
+ break;
+ }
+ }
+
+ if (dependent_srcs || writes_overlap) {
+ break;
+ }
+ }
+
+ return !writes_overlap && !dependent_srcs;
+ }
+
+ ~ggml_cuda_concurrent_event() {
+ if (fork_event != nullptr) {
+ CUDA_CHECK(cudaEventDestroy(fork_event));
+ }
+ for (cudaEvent_t e : join_events) {
+ if (e != nullptr) {
+ CUDA_CHECK(cudaEventDestroy(e));
+ }
+ }
+ }
+};
+
+struct ggml_cuda_stream_context {
+ std::unordered_map<const ggml_tensor *, ggml_cuda_concurrent_event> concurrent_events;
+
+ void reset() {
+ concurrent_events.clear();
+ }
+};
+
+struct ggml_backend_cuda_context {
+ int device;
+ std::string name;
+ cudaEvent_t copy_event = nullptr;
+
+ cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};
+ cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};
+ void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};
+ size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {};
+
+ int curr_stream_no = 0;
+
+#ifdef USE_CUDA_GRAPH
+ // Map from first_node_ptr to cuda_graph - allows multiple graphs per context
+ // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe)
+ std::unordered_map<const void *, std::unique_ptr<ggml_cuda_graph>> cuda_graphs;
+
+ int64_t last_graph_eviction_sweep = 0;
+
+ ggml_cuda_graph * cuda_graph(const void * first_node_ptr) {
+ const int64_t time_now = ggml_time_us();
+
+ // sweep every 5s, evicting cuda graphs unused for >=10s
+ if (time_now - last_graph_eviction_sweep >= 5'000'000) {
+ last_graph_eviction_sweep = time_now;
+ for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) {
+ if (time_now - it->second->last_used_time >= 10'000'000) {
+ it = cuda_graphs.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+
+ auto it = cuda_graphs.find(first_node_ptr);
+ if (it == cuda_graphs.end()) {
+ it = cuda_graphs.emplace(first_node_ptr, std::make_unique<ggml_cuda_graph>()).first;
+ }
+ it->second->last_used_time = time_now;
+ return it->second.get();
+ }
+
+ // Check if any CUDA graph is enabled for this context (used by kernels that need to know
+ // if graphs are in use without having access to the specific graph key)
+ bool any_cuda_graph_enabled() const {
+ for (const auto & [key, graph] : cuda_graphs) {
+ if (graph && graph->is_enabled()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Check if any CUDA graph has an instance for this context
+ bool any_cuda_graph_has_instance() const {
+ for (const auto & [key, graph] : cuda_graphs) {
+ if (graph && graph->instance != nullptr) {
+ return true;
+ }
+ }
+ return false;
+ }
+#endif // USE_CUDA_GRAPH
+
+ explicit ggml_backend_cuda_context(int device) :
+ device(device),
+ name(GGML_CUDA_NAME + std::to_string(device)) {
+ }
+
+ ggml_cuda_stream_context concurrent_stream_context;
+
+ ~ggml_backend_cuda_context();
+
+ cudaStream_t stream(int device, int stream) {
+ if (streams[device][stream] == nullptr) {
+ ggml_cuda_set_device(device);
+ CUDA_CHECK(cudaStreamCreateWithFlags(&streams[device][stream], cudaStreamNonBlocking));
+ }
+ return streams[device][stream];
+ }
+
+ cudaStream_t stream() { return stream(device, curr_stream_no); }
+
+ ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; }
+
+ cublasHandle_t cublas_handle() {
+ if (cublas_handles[device][curr_stream_no] == nullptr) {
+ ggml_cuda_set_device(device);
+ CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no]));
+ CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH));
+ CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream()));
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2))
+ if (cublas_workspace_sizes[device] == 0) {
+ const int cc = ggml_cuda_info().devices[device].cc;
+ cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024;
+ }
+ CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device]));
+ CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device]));
+#endif
+ }
+ return cublas_handles[device][curr_stream_no];
+ }
+
+ // pool
+ std::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS];
+
+ static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(int device, int stream_no);
+
+ ggml_cuda_pool & pool(int device) {
+ if (pools[device][curr_stream_no] == nullptr) {
+ pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no);
+ }
+ return *pools[device][curr_stream_no];
+ }
+
+ ggml_cuda_pool & pool() {
+ return pool(device);
+ }
+};
+
+struct ggml_cuda_mm_fusion_args_host {
+ const ggml_tensor * x_bias = nullptr;
+ const ggml_tensor * gate = nullptr;
+ const ggml_tensor * gate_bias = nullptr;
+ const ggml_tensor * x_scale = nullptr;
+ const ggml_tensor * gate_scale = nullptr;
+ ggml_glu_op glu_op;
+ float glu_limit = 0.0f;
+ const ggml_tensor * shared_up = nullptr;
+ const ggml_tensor * shared_gate = nullptr;
+ ggml_tensor * shared_dst = nullptr;
+};
+struct ggml_cuda_mm_fusion_args_device {
+ const void * x_bias = nullptr;
+ const void * gate = nullptr;
+ const void * gate_bias = nullptr;
+ const void * x_scale = nullptr;
+ const void * gate_scale = nullptr;
+ ggml_glu_op glu_op;
+ float glu_limit = 0.0f;
+ const void * shared_up = nullptr;
+ const void * shared_gate = nullptr;
+ float * shared_dst = nullptr;
+ uint32_t shared_stride_col_dst = 0;
+};
+
+struct ggml_cuda_kernel_launch_params {
+ dim3 block_nums;
+ dim3 block_dims;
+ size_t shmem;
+ cudaStream_t stream;
+
+ // size_t shmem
+ ggml_cuda_kernel_launch_params(const dim3& block_nums_, const dim3& block_dims_, const size_t shmem_, const cudaStream_t stream_)
+ : block_nums(block_nums_), block_dims(block_dims_), shmem(shmem_), stream(stream_) {}
+
+ // Some call sites pass ints instead of the required size_t. This 2nd constructor casts int->size_t to avoid these -Wnarrowing warnings.
+ ggml_cuda_kernel_launch_params(const dim3& block_nums_, const dim3& block_dims_, const int shmem_, const cudaStream_t stream_)
+ : block_nums(block_nums_), block_dims(block_dims_), shmem((size_t)shmem_), stream(stream_) {}
+};
+
+#if defined(GGML_CUDA_USE_PDL)
+struct ggml_cuda_pdl_config {
+ cudaLaunchAttribute attr;
+ cudaLaunchConfig_t cfg;
+
+ ggml_cuda_pdl_config(const ggml_cuda_kernel_launch_params & params) {
+ attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;
+ attr.val.programmaticStreamSerializationAllowed = 1;
+
+ cfg = {};
+ cfg.gridDim = params.block_nums;
+ cfg.blockDim = params.block_dims;
+ cfg.dynamicSmemBytes = params.shmem;
+ cfg.stream = params.stream;
+ cfg.attrs = &attr;
+ cfg.numAttrs = 1;
+ }
+
+ // Delete due to &attr
+ ggml_cuda_pdl_config(const ggml_cuda_pdl_config&) = delete;
+ ggml_cuda_pdl_config& operator=(const ggml_cuda_pdl_config&) = delete;
+ ggml_cuda_pdl_config& operator=(ggml_cuda_pdl_config&&) = delete;
+
+};
+
+static bool ggml_cuda_kernel_can_use_pdl(const void * kernel) {
+ const int device = ggml_cuda_get_device();
+
+ struct cache_key {
+ int device;
+ const void * kernel;
+
+ bool operator==(const cache_key & other) const { return device == other.device && kernel == other.kernel; }
+ };
+
+ struct cache_key_hash {
+ // MurmurHash3 mixing function for better hash distribution (vs. just std::hash which in some implementations simply returns the identity)
+ static size_t hash_mix(size_t x) {
+ std::uint64_t y = x;
+ const std::uint64_t m = 0xe9846af9b1a615d;
+
+ y ^= y >> 32;
+ y *= m;
+ y ^= y >> 32;
+ y *= m;
+ y ^= y >> 28;
+
+ return static_cast<size_t>(y);
+ }
+
+ size_t operator()(const cache_key & key) const {
+ // Use a nonzero seed to avoid mapping all-zero keys to zero
+ size_t h = 42;
+ h = hash_mix(h + key.device);
+ h = hash_mix(h + reinterpret_cast<size_t>(key.kernel));
+ return h;
+ }
+ };
+
+ static std::mutex cache_mutex;
+ static std::unordered_map<cache_key, bool, cache_key_hash> cache;
+
+ const cache_key key = { device, kernel };
+ std::lock_guard<std::mutex> lock(cache_mutex);
+ const auto it = cache.find(key);
+ if (it != cache.end()) {
+ return it->second;
+ }
+
+ cudaFuncAttributes attr = {};
+ CUDA_CHECK(cudaFuncGetAttributes(&attr, kernel));
+
+ // PDL device-side primitives are emitted only for PTX versions >= 90.
+ // We have to guard on a loaded kernel's PTX version so a kernel forward-JIT'ed
+ // from pre-Hopper PTX to a Hopper-or-newer GPU does not opt into PDL.
+ const bool can_use_pdl = attr.ptxVersion >= 90;
+ cache.emplace(key, can_use_pdl);
+ return can_use_pdl;
+}
+
+#endif //defined(GGML_CUDA_USE_PDL)
+
+// PDL and __restrict__ need to be mutually exclusive, see https://github.com/ggml-org/llama.cpp/pull/24030
+# if (defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER)
+# define GGML_CUDA_RESTRICT
+# else
+# define GGML_CUDA_RESTRICT __restrict__
+# endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER
+
+template<typename Kernel, typename... Args>
+static __inline__ void ggml_cuda_kernel_launch(Kernel kernel, const ggml_cuda_kernel_launch_params & launch_params, Args&&... args) {
+#if defined(GGML_CUDA_USE_PDL)
+
+ static const bool env_pdl_enabled = []() {
+ const char * env = getenv("GGML_CUDA_PDL");
+ return env == nullptr || std::atoi(env) != 0;
+ }();
+
+ if (env_pdl_enabled && ggml_cuda_kernel_can_use_pdl(reinterpret_cast<const void *>(kernel))) {
+ auto pdl_cfg = ggml_cuda_pdl_config(launch_params);
+
+ CUDA_CHECK(cudaLaunchKernelEx(&pdl_cfg.cfg, kernel, std::forward<Args>(args)... ));
+ return;
+ }
+#endif //defined(GGML_CUDA_USE_PDL)
+
+ kernel<<<launch_params.block_nums, launch_params.block_dims, launch_params.shmem, launch_params.stream>>>(std::forward<Args>(args)... );
+ CUDA_CHECK(cudaGetLastError());
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cu
new file mode 100644
index 00000000..77fd2889
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cu
@@ -0,0 +1,743 @@
+#include "convert.cuh"
+#include "dequantize.cuh"
+
+#include <cstdint>
+
+#define CUDA_Q8_0_NE_ALIGN 2048
+
+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
+static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y,
+ const int64_t ne00, const int64_t ne01,
+ const int64_t ne0203, const uint3 ne02,
+ const int64_t s01, const int64_t s02, const int64_t s03) {
+ const int64_t i00 = 2 * (int64_t(blockDim.x)*blockIdx.x + threadIdx.x);
+
+ if (i00 >= ne00) {
+ return;
+ }
+
+ for (int64_t i01 = blockIdx.y; i01 < ne01; i01 += gridDim.y) {
+ for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {
+ const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);
+ const int64_t i02 = dm.y;
+ const int64_t i03 = dm.x;
+
+ const int64_t ibx0 = i03*s03 + i02*s02 + i01*s01;
+
+ const int64_t ib = ibx0 + i00/qk; // block index
+ const int64_t iqs = (i00%qk)/qr; // quant index
+ const int64_t iybs = i00 - i00%qk; // y block start index
+ const int64_t y_offset = qr == 1 ? 1 : qk/2;
+
+ // dequantize
+ float2 v;
+ dequantize_kernel(vx, ib, iqs, v);
+
+ const int64_t iy0 = (i0203*ne01 + i01)*ne00 + iybs + iqs;
+ y[iy0 + 0] = ggml_cuda_cast<dst_t>(v.x);
+ y[iy0 + y_offset] = ggml_cuda_cast<dst_t>(v.y);
+ }
+ }
+}
+
+template <bool need_check>
+static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int64_t k) {
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL
+ constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;
+
+ const int64_t i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;
+ const int * x0 = ((int *) vx) + blockIdx.x * nint;
+ half2 * y2 = (half2 *) (y + i0);
+
+ __shared__ int vals[nint];
+
+#pragma unroll
+ for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {
+ if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {
+ break;
+ }
+
+ const int ix = ix0 + threadIdx.x;
+ vals[ix] = x0[ix];
+ }
+
+ __syncthreads();
+
+#pragma unroll
+ for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {
+ if (need_check && i0 + iy + 2*threadIdx.x >= k) {
+ return;
+ }
+
+ const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);
+ const half d = *b0;
+ const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];
+
+ y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));
+ }
+#else
+ GGML_UNUSED_VARS(vx, y, k);
+ NO_DEVICE_CODE;
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {
+
+ const int64_t i = blockIdx.x;
+
+ // assume 32 threads
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
+ const int64_t ib = 8*i + ir;
+ if (ib >= nb32) {
+ return;
+ }
+
+ dst_t * y = yy + 256*i + 32*ir + 4*il;
+
+ const block_q4_0 * x = (const block_q4_0 *)vx + ib;
+ const float d = __half2float(x->d);
+ const float dm = -8*d;
+
+ const uint8_t * q = x->qs + 4*il;
+
+ for (int l = 0; l < 4; ++l) {
+ y[l+ 0] = ggml_cuda_cast<dst_t>(d * (q[l] & 0xF) + dm);
+ y[l+16] = ggml_cuda_cast<dst_t>(d * (q[l] >> 4) + dm);
+ }
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {
+
+ const int64_t i = blockIdx.x;
+
+ // assume 32 threads
+ const int64_t tid = threadIdx.x;
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
+ const int64_t ib = 8*i + ir;
+ if (ib >= nb32) {
+ return;
+ }
+
+ dst_t * y = yy + 256*i + 32*ir + 4*il;
+
+ const block_q4_1 * x = (const block_q4_1 *)vx + ib;
+ const float2 d = __half22float2(x->dm);
+
+ const uint8_t * q = x->qs + 4*il;
+
+ for (int l = 0; l < 4; ++l) {
+ y[l+ 0] = ggml_cuda_cast<dst_t>(d.x * (q[l] & 0xF) + d.y);
+ y[l+16] = ggml_cuda_cast<dst_t>(d.x * (q[l] >> 4) + d.y);
+ }
+}
+
+//================================== k-quants
+
+template<typename dst_t>
+static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_q2_K(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_q3_K(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_q4_K(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_q5_K(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_q6_K(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_iq2_xxs(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_iq2_xs(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_iq2_s(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_iq3_xxs(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_iq3_s(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_iq1_s(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_iq1_m(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {
+ const int64_t i = blockIdx.x;
+
+ const int64_t ib = 8*i + threadIdx.x%8;
+ if (ib >= nb32) {
+ return;
+ }
+
+ dequantize_iq4_nl(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_iq4_xs(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template<typename dst_t>
+static __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy) {
+ const int64_t i = blockIdx.x;
+
+ dequantize_mxfp4(vx, i, yy + i*QK_K, threadIdx.x);
+}
+
+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
+static void dequantize_block_cuda(const void * vx, dst_t * y,
+ const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
+ const int64_t s01, const int64_t s02, const int64_t s03, cudaStream_t stream) {
+ const int64_t ne0203 = ne02*ne03;
+ const uint3 ne02_fdv = init_fastdiv_values(ne02);
+ const dim3 num_blocks((ne00 + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE), (int)std::min(ne01, (int64_t)65535), (int)std::min(ne0203, (int64_t)65535));
+ dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>
+ (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);
+}
+
+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
+static void dequantize_block_cont_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {
+ dequantize_block_cuda<qk, qr, dequantize_kernel, dst_t>(vx, y, k, 1, 1, 1, k/qk, k/qk, k/qk, stream);
+}
+
+static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int64_t k, cudaStream_t stream) {
+ const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;
+ if (k % CUDA_Q8_0_NE_ALIGN == 0) {
+ const bool need_check = false;
+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);
+ } else {
+ const bool need_check = true;
+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);
+ }
+}
+
+template<typename dst_t>
+static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb32 = k / 32;
+ const int nb = (k + 255) / 256;
+ dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);
+}
+
+template<typename dst_t>
+static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb32 = k / 32;
+ const int nb = (k + 255) / 256;
+ dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);
+}
+
+template<typename dst_t>
+static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb32 = k / QK4_NL;
+ const int nb = (k + QK_K - 1) / QK_K;
+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y, nb32);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = k / QK_K;
+ dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = (k + QK_K - 1) / QK_K;
+ dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template<typename dst_t>
+static void dequantize_row_mxfp4_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ const int nb = (k + QK_K - 1) / QK_K;
+ dequantize_block_mxfp4<<<nb, 32, 0, stream>>>(vx, y);
+}
+
+template <typename dst_t>
+static __global__ void dequantize_block_nvfp4(
+ const void * __restrict__ vx,
+ dst_t * __restrict__ yy,
+ const int64_t ne) {
+ const int64_t i = blockIdx.x;
+ const int tid = threadIdx.x;
+
+ const int64_t base = i * QK_NVFP4;
+ if (base >= ne) {
+ return;
+ }
+
+ const block_nvfp4 * x = (const block_nvfp4 *) vx;
+ const block_nvfp4 & xb = x[i];
+
+ const int sub = tid / (QK_NVFP4_SUB / 2);
+ const int j = tid % (QK_NVFP4_SUB / 2);
+
+ const float d = ggml_cuda_ue4m3_to_fp32(xb.d[sub]);
+ const uint8_t q = xb.qs[sub * (QK_NVFP4_SUB / 2) + j];
+
+ const int64_t y0 = base + sub * QK_NVFP4_SUB + j;
+ const int64_t y1 = y0 + QK_NVFP4_SUB / 2;
+
+ yy[y0] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q & 0x0F]);
+ yy[y1] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q >> 4]);
+}
+
+template <typename dst_t>
+static void dequantize_row_nvfp4_cuda(
+ const void * vx,
+ dst_t * y,
+ const int64_t k,
+ cudaStream_t stream) {
+ GGML_ASSERT(k % QK_NVFP4 == 0);
+ const int nb = k / QK_NVFP4;
+ dequantize_block_nvfp4<<<nb, 32, 0, stream>>>(vx, y, k);
+}
+template <typename src_t, typename dst_t>
+static __global__ void convert_unary(
+ const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t ne00, const int64_t ne01,
+ const int64_t ne0203, const uint3 ne02,
+ const int64_t s01, const int64_t s02, const int64_t s03) {
+ const int64_t i00 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i00 >= ne00) {
+ return;
+ }
+
+ const src_t * x = (const src_t *) vx;
+
+ for (int64_t i01 = blockIdx.y; i01 < ne01; i01 += gridDim.y) {
+ for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {
+ const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);
+ const int64_t i02 = dm.y;
+ const int64_t i03 = dm.x;
+
+ const int64_t ix = i03*s03 + i02*s02 + i01*s01 + i00;
+ const int64_t iy = (i0203*ne01 + i01)*ne00 + i00;
+ y[iy] = ggml_cuda_cast<dst_t>(x[ix]);
+ }
+ }
+}
+
+template <typename T> struct alignas(sizeof(T)*4) cvt_vec4 { T v[4]; };
+
+// four elements per thread, so a warp moves 512B (RDNA) / 1k (CDNA) per load
+template <typename src_t, typename dst_t>
+static __global__ void convert_unary_cont_vec4(
+ const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k4) {
+ const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i >= k4) {
+ return;
+ }
+
+ const cvt_vec4<src_t> xv = ((const cvt_vec4<src_t> *) vx)[i];
+
+ cvt_vec4<dst_t> yv;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ yv.v[j] = ggml_cuda_cast<dst_t>(xv.v[j]);
+ }
+
+ ((cvt_vec4<dst_t> *) y)[i] = yv;
+}
+
+template <typename src_t, typename dst_t>
+static void convert_unary_cuda(const void * vx, dst_t * y,
+ const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
+ const int64_t s01, const int64_t s02, const int64_t s03, cudaStream_t stream) {
+ const int64_t ne0203 = ne02*ne03;
+ const uint3 ne02_fdv = init_fastdiv_values(ne02);
+ const dim3 num_blocks((ne00 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE, (int)std::min(ne01, (int64_t)65535), (int)std::min(ne0203, (int64_t)65535));
+ convert_unary<src_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>
+ (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);
+}
+
+template <typename src_t, typename dst_t>
+static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+ if (k % 4 == 0 &&
+ (uintptr_t) vx % alignof(cvt_vec4<src_t>) == 0 &&
+ (uintptr_t) y % alignof(cvt_vec4<dst_t>) == 0) {
+ const int64_t k4 = k/4;
+ const int64_t num_blocks = (k4 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;
+ convert_unary_cont_vec4<src_t, dst_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k4);
+ return;
+ }
+
+ convert_unary_cuda<src_t>(vx, y, k, 1, 1, 1, k, k, k, stream);
+}
+
+to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_Q1_0:
+ return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;
+ case GGML_TYPE_Q2_0:
+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;
+ case GGML_TYPE_Q4_0:
+ return dequantize_row_q4_0_cuda;
+ case GGML_TYPE_Q4_1:
+ return dequantize_row_q4_1_cuda;
+ case GGML_TYPE_Q5_0:
+ return dequantize_block_cont_cuda<QK5_0, QR5_0, dequantize_q5_0>;
+ case GGML_TYPE_Q5_1:
+ return dequantize_block_cont_cuda<QK5_1, QR5_1, dequantize_q5_1>;
+ case GGML_TYPE_Q8_0:
+ return dequantize_block_cont_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_Q2_K:
+ return dequantize_row_q2_K_cuda;
+ case GGML_TYPE_Q3_K:
+ return dequantize_row_q3_K_cuda;
+ case GGML_TYPE_Q4_K:
+ return dequantize_row_q4_K_cuda;
+ case GGML_TYPE_Q5_K:
+ return dequantize_row_q5_K_cuda;
+ case GGML_TYPE_Q6_K:
+ return dequantize_row_q6_K_cuda;
+ case GGML_TYPE_IQ2_XXS:
+ return dequantize_row_iq2_xxs_cuda;
+ case GGML_TYPE_IQ2_XS:
+ return dequantize_row_iq2_xs_cuda;
+ case GGML_TYPE_IQ2_S:
+ return dequantize_row_iq2_s_cuda;
+ case GGML_TYPE_IQ3_XXS:
+ return dequantize_row_iq3_xxs_cuda;
+ case GGML_TYPE_IQ1_S:
+ return dequantize_row_iq1_s_cuda;
+ case GGML_TYPE_IQ1_M:
+ return dequantize_row_iq1_m_cuda;
+ case GGML_TYPE_IQ4_NL:
+ return dequantize_row_iq4_nl_cuda;
+ case GGML_TYPE_IQ4_XS:
+ return dequantize_row_iq4_xs_cuda;
+ case GGML_TYPE_IQ3_S:
+ return dequantize_row_iq3_s_cuda;
+ case GGML_TYPE_MXFP4:
+ return dequantize_row_mxfp4_cuda;
+ case GGML_TYPE_NVFP4:
+ return dequantize_row_nvfp4_cuda;
+ case GGML_TYPE_F32:
+ return convert_unary_cont_cuda<float>;
+ case GGML_TYPE_F16:
+ return convert_unary_cont_cuda<half>;
+ default:
+ return nullptr;
+ }
+}
+
+to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_Q1_0:
+ return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;
+ case GGML_TYPE_Q2_0:
+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;
+ case GGML_TYPE_Q4_0:
+ return dequantize_row_q4_0_cuda;
+ case GGML_TYPE_Q4_1:
+ return dequantize_row_q4_1_cuda;
+ case GGML_TYPE_Q5_0:
+ return dequantize_block_cont_cuda<QK5_0, QR5_0, dequantize_q5_0>;
+ case GGML_TYPE_Q5_1:
+ return dequantize_block_cont_cuda<QK5_1, QR5_1, dequantize_q5_1>;
+ case GGML_TYPE_Q8_0:
+ if (fp16_available(ggml_cuda_info().devices[ggml_cuda_get_device()].cc)) {
+ return dequantize_block_q8_0_f16_cuda;
+ }
+ return dequantize_block_cont_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_Q2_K:
+ return dequantize_row_q2_K_cuda;
+ case GGML_TYPE_Q3_K:
+ return dequantize_row_q3_K_cuda;
+ case GGML_TYPE_Q4_K:
+ return dequantize_row_q4_K_cuda;
+ case GGML_TYPE_Q5_K:
+ return dequantize_row_q5_K_cuda;
+ case GGML_TYPE_Q6_K:
+ return dequantize_row_q6_K_cuda;
+ case GGML_TYPE_IQ2_XXS:
+ return dequantize_row_iq2_xxs_cuda;
+ case GGML_TYPE_IQ2_XS:
+ return dequantize_row_iq2_xs_cuda;
+ case GGML_TYPE_IQ2_S:
+ return dequantize_row_iq2_s_cuda;
+ case GGML_TYPE_IQ3_XXS:
+ return dequantize_row_iq3_xxs_cuda;
+ case GGML_TYPE_IQ1_S:
+ return dequantize_row_iq1_s_cuda;
+ case GGML_TYPE_IQ1_M:
+ return dequantize_row_iq1_m_cuda;
+ case GGML_TYPE_IQ4_NL:
+ return dequantize_row_iq4_nl_cuda;
+ case GGML_TYPE_IQ4_XS:
+ return dequantize_row_iq4_xs_cuda;
+ case GGML_TYPE_IQ3_S:
+ return dequantize_row_iq3_s_cuda;
+ case GGML_TYPE_MXFP4:
+ return dequantize_row_mxfp4_cuda;
+ case GGML_TYPE_NVFP4:
+ return dequantize_row_nvfp4_cuda;
+ case GGML_TYPE_F32:
+ return convert_unary_cont_cuda<float>;
+ case GGML_TYPE_BF16:
+ return convert_unary_cont_cuda<nv_bfloat16>;
+ default:
+ return nullptr;
+ }
+}
+
+to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_Q1_0:
+ return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;
+ case GGML_TYPE_Q2_0:
+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;
+ case GGML_TYPE_Q4_0:
+ return dequantize_row_q4_0_cuda;
+ case GGML_TYPE_Q4_1:
+ return dequantize_row_q4_1_cuda;
+ case GGML_TYPE_Q5_0:
+ return dequantize_block_cont_cuda<QK5_0, QR5_0, dequantize_q5_0>;
+ case GGML_TYPE_Q5_1:
+ return dequantize_block_cont_cuda<QK5_1, QR5_1, dequantize_q5_1>;
+ case GGML_TYPE_Q8_0:
+ return dequantize_block_cont_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_Q2_K:
+ return dequantize_row_q2_K_cuda;
+ case GGML_TYPE_Q3_K:
+ return dequantize_row_q3_K_cuda;
+ case GGML_TYPE_Q4_K:
+ return dequantize_row_q4_K_cuda;
+ case GGML_TYPE_Q5_K:
+ return dequantize_row_q5_K_cuda;
+ case GGML_TYPE_Q6_K:
+ return dequantize_row_q6_K_cuda;
+ case GGML_TYPE_IQ2_XXS:
+ return dequantize_row_iq2_xxs_cuda;
+ case GGML_TYPE_IQ2_XS:
+ return dequantize_row_iq2_xs_cuda;
+ case GGML_TYPE_IQ2_S:
+ return dequantize_row_iq2_s_cuda;
+ case GGML_TYPE_IQ3_XXS:
+ return dequantize_row_iq3_xxs_cuda;
+ case GGML_TYPE_IQ1_S:
+ return dequantize_row_iq1_s_cuda;
+ case GGML_TYPE_IQ1_M:
+ return dequantize_row_iq1_m_cuda;
+ case GGML_TYPE_IQ4_NL:
+ return dequantize_row_iq4_nl_cuda;
+ case GGML_TYPE_IQ4_XS:
+ return dequantize_row_iq4_xs_cuda;
+ case GGML_TYPE_IQ3_S:
+ return dequantize_row_iq3_s_cuda;
+ case GGML_TYPE_MXFP4:
+ return dequantize_row_mxfp4_cuda;
+ case GGML_TYPE_NVFP4:
+ return dequantize_row_nvfp4_cuda;
+ case GGML_TYPE_F16:
+ return convert_unary_cont_cuda<half>;
+ case GGML_TYPE_BF16:
+ return convert_unary_cont_cuda<nv_bfloat16>;
+ default:
+ return nullptr;
+ }
+}
+
+to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_F32:
+ return convert_unary_cuda<float>;
+ case GGML_TYPE_Q1_0:
+ return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;
+ case GGML_TYPE_Q2_0:
+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;
+ case GGML_TYPE_Q4_0:
+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
+ case GGML_TYPE_Q4_1:
+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;
+ case GGML_TYPE_Q5_0:
+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
+ case GGML_TYPE_Q5_1:
+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
+ case GGML_TYPE_Q8_0:
+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_BF16:
+ return convert_unary_cuda<nv_bfloat16>;
+ default:
+ return nullptr;
+ }
+}
+
+to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_F32:
+ return convert_unary_cuda<float, nv_bfloat16>;
+ case GGML_TYPE_Q1_0:
+ return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;
+ case GGML_TYPE_Q2_0:
+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;
+ case GGML_TYPE_Q4_0:
+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
+ case GGML_TYPE_Q4_1:
+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;
+ case GGML_TYPE_Q5_0:
+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
+ case GGML_TYPE_Q5_1:
+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
+ case GGML_TYPE_Q8_0:
+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_F16:
+ return convert_unary_cuda<half, nv_bfloat16>;
+ default:
+ return nullptr;
+ }
+}
+
+to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_F16:
+ return convert_unary_cuda<half, float>;
+ case GGML_TYPE_Q1_0:
+ return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;
+ case GGML_TYPE_Q2_0:
+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;
+ case GGML_TYPE_Q4_0:
+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
+ case GGML_TYPE_Q4_1:
+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;
+ case GGML_TYPE_Q5_0:
+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
+ case GGML_TYPE_Q5_1:
+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
+ case GGML_TYPE_Q8_0:
+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_BF16:
+ return convert_unary_cuda<nv_bfloat16, float>;
+ default:
+ return nullptr;
+ }
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cuh
new file mode 100644
index 00000000..91255ccf
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cuh
@@ -0,0 +1,66 @@
+#pragma once
+#include "common.cuh"
+
+#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
+
+template<typename T>
+using to_t_cuda_t = void (*)(const void * x, T * y, int64_t k, cudaStream_t stream);
+
+typedef to_t_cuda_t<float> to_fp32_cuda_t;
+typedef to_t_cuda_t<half> to_fp16_cuda_t;
+typedef to_t_cuda_t<nv_bfloat16> to_bf16_cuda_t;
+
+to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);
+
+to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type);
+
+to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);
+
+// TODO more general support for non-contiguous inputs
+
+template<typename T>
+using to_t_nc_cuda_t = void (*)(const void * x, T * y,
+ int64_t ne00, int64_t ne01, int64_t ne02, int64_t ne03,
+ int64_t s01, int64_t s02, int64_t s03, cudaStream_t stream);
+
+typedef to_t_nc_cuda_t<float> to_fp32_nc_cuda_t;
+typedef to_t_nc_cuda_t<half> to_fp16_nc_cuda_t;
+typedef to_t_nc_cuda_t<nv_bfloat16> to_bf16_nc_cuda_t;
+
+to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type);
+to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type);
+to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type);
+
+template<typename dst_t, typename src_t>
+ __host__ __device__ inline dst_t ggml_cuda_cast(src_t x) {
+ if constexpr (std::is_same_v<dst_t, src_t>) {
+ return x;
+ } else if constexpr(std::is_same_v<dst_t, nv_bfloat16>) {
+ return __float2bfloat16(float(x));
+ } else if constexpr(std::is_same_v<src_t, nv_bfloat16>) {
+ return __bfloat162float(x);
+ } else if constexpr(std::is_same_v<src_t, float2> && std::is_same_v<dst_t, half2>) {
+ return __float22half2_rn(x);
+ } else if constexpr(std::is_same_v<src_t, nv_bfloat162> && std::is_same_v<dst_t, float2>) {
+#ifdef GGML_USE_HIP
+ return make_float2(__bfloat162float(__low2bfloat16(x)), __bfloat162float(__high2bfloat16(x)));
+#else
+#if !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= 800
+ return __bfloat1622float2(x);
+#else
+ return make_float2(__bfloat162float(x.x), __bfloat162float(x.y));
+#endif // !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= 800
+#endif // GGML_USE_HIP
+ } else if constexpr(std::is_same_v<src_t, float2> && std::is_same_v<dst_t, nv_bfloat162>) {
+ // bypass compile error on cuda 12.0.1
+#ifdef GGML_USE_HIP
+ return __float22bfloat162_rn(x);
+#else
+ return {x.x, x.y};
+#endif // GGML_USE_HIP
+ } else if constexpr(std::is_same_v<dst_t, int32_t>) {
+ return int32_t(x);
+ } else {
+ return float(x);
+ }
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/dequantize.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/dequantize.cuh
new file mode 100644
index 00000000..651524be
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/dequantize.cuh
@@ -0,0 +1,452 @@
+#include "common.cuh"
+#include "convert.cuh"
+
+static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
+ const block_q1_0 * x = (const block_q1_0 *) vx;
+
+ const float d = x[ib].d;
+
+ const int bit_index_0 = iqs;
+ const int bit_index_1 = iqs + 1;
+
+ const int byte_index_0 = bit_index_0 / 8;
+ const int bit_offset_0 = bit_index_0 % 8;
+
+ const int byte_index_1 = bit_index_1 / 8;
+ const int bit_offset_1 = bit_index_1 % 8;
+
+ // Extract bits: 1 = +d, 0 = -d (branchless)
+ const int bit_0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 1;
+ const int bit_1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 1;
+
+ v.x = (2*bit_0 - 1) * d;
+ v.y = (2*bit_1 - 1) * d;
+}
+
+static __device__ __forceinline__ void dequantize_q2_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
+ const block_q2_0 * x = (const block_q2_0 *) vx;
+
+ const float d = x[ib].d;
+
+ // Q2_0: 2 bits per element, 4 elements per byte.
+ // Stored code c in {0,1,2,3} maps to symbol s = c - 1 in {-1, 0, +1, +2}.
+ const int byte_index_0 = iqs / 4;
+ const int bit_offset_0 = (iqs % 4) * 2;
+
+ const int byte_index_1 = (iqs + 1) / 4;
+ const int bit_offset_1 = ((iqs + 1) % 4) * 2;
+
+ const int c0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 0x3;
+ const int c1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 0x3;
+
+ v.x = (c0 - 1) * d;
+ v.y = (c1 - 1) * d;
+}
+
+static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
+ const block_q4_0 * x = (const block_q4_0 *) vx;
+
+ const float d = x[ib].d;
+
+ const int vui = x[ib].qs[iqs];
+
+ v.x = vui & 0xF;
+ v.y = vui >> 4;
+
+ v.x = (v.x - 8.0f) * d;
+ v.y = (v.y - 8.0f) * d;
+}
+
+static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, float2 & v){
+ const block_q4_1 * x = (const block_q4_1 *) vx;
+
+ const float2 dm = __half22float2(x[ib].dm);
+
+ const int vui = x[ib].qs[iqs];
+
+ v.x = vui & 0xF;
+ v.y = vui >> 4;
+
+ v.x = (v.x * dm.x) + dm.y;
+ v.y = (v.y * dm.x) + dm.y;
+}
+
+static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
+ const block_q5_0 * x = (const block_q5_0 *) vx;
+
+ const float d = x[ib].d;
+
+ uint32_t qh;
+ memcpy(&qh, x[ib].qh, sizeof(qh));
+
+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
+
+ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);
+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);
+
+ v.x = (v.x - 16.0f) * d;
+ v.y = (v.y - 16.0f) * d;
+}
+
+static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, float2 & v){
+ const block_q5_1 * x = (const block_q5_1 *) vx;
+
+ const float2 dm = __half22float2(x[ib].dm);
+
+ uint32_t qh;
+ memcpy(&qh, x[ib].qh, sizeof(qh));
+
+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
+
+ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);
+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);
+
+ v.x = (v.x * dm.x) + dm.y;
+ v.y = (v.y * dm.x) + dm.y;
+}
+
+static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
+ const block_q8_0 * x = (const block_q8_0 *) vx;
+
+ const float d = x[ib].d;
+
+ v.x = x[ib].qs[iqs + 0];
+ v.y = x[ib].qs[iqs + 1];
+
+ v.x *= d;
+ v.y *= d;
+}
+
+//================================== k-quants
+
+// Each call dequantizes one super-block of QK_K values into y using the
+// thread layout of the caller: 32 threads for q4_K, 64 threads otherwise.
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_q2_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {
+ const block_q2_K * x = (const block_q2_K *) vx;
+
+ const int64_t n = tid/32;
+ const int64_t l = tid - 32*n;
+ const int64_t is = 8*n + l/16;
+
+ const uint8_t q = x[ib].qs[32*n + l];
+ dst_t * y = yy + 128*n;
+
+ float dall = __low2half(x[ib].dm);
+ float dmin = __high2half(x[ib].dm);
+ y[l+ 0] = ggml_cuda_cast<dst_t>(dall * (x[ib].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[ib].scales[is+0] >> 4));
+ y[l+32] = ggml_cuda_cast<dst_t>(dall * (x[ib].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[ib].scales[is+2] >> 4));
+ y[l+64] = ggml_cuda_cast<dst_t>(dall * (x[ib].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[ib].scales[is+4] >> 4));
+ y[l+96] = ggml_cuda_cast<dst_t>(dall * (x[ib].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[ib].scales[is+6] >> 4));
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_q3_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {
+ const block_q3_K * x = (const block_q3_K *) vx;
+
+ const int64_t r = tid/4;
+ const int64_t t = r/2;
+ const int64_t is0 = r%2;
+ const int64_t l0 = 16*is0 + 4*(tid%4);
+ const int64_t n = t / 4;
+ const int64_t j = t - 4*n;
+
+ uint8_t m = 1 << (4*n + j);
+ int64_t is = 8*n + 2*j + is0;
+ int shift = 2*j;
+
+ int8_t us = is < 4 ? (x[ib].scales[is-0] & 0xF) | (((x[ib].scales[is+8] >> 0) & 3) << 4) :
+ is < 8 ? (x[ib].scales[is-0] & 0xF) | (((x[ib].scales[is+4] >> 2) & 3) << 4) :
+ is < 12 ? (x[ib].scales[is-8] >> 4) | (((x[ib].scales[is+0] >> 4) & 3) << 4) :
+ (x[ib].scales[is-8] >> 4) | (((x[ib].scales[is-4] >> 6) & 3) << 4);
+ float d_all = x[ib].d;
+ float dl = d_all * (us - 32);
+
+ dst_t * y = yy + 128*n + 32*j;
+ const uint8_t * q = x[ib].qs + 32*n;
+ const uint8_t * hm = x[ib].hmask;
+
+ for (int l = l0; l < l0+4; ++l) {
+ y[l] = ggml_cuda_cast<dst_t>(dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)));
+ }
+}
+
+static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {
+ if (j < 4) {
+ d = q[j] & 63; m = q[j + 4] & 63;
+ } else {
+ d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);
+ m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_q4_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {
+ const block_q4_K * x = (const block_q4_K *) vx;
+
+ // assume 32 threads
+ const int64_t il = tid/8;
+ const int64_t ir = tid%8;
+ const int64_t is = 2*il;
+ const int64_t n = 4;
+
+ dst_t * y = yy + 64*il + n*ir;
+
+ const float dall = __low2half(x[ib].dm);
+ const float dmin = __high2half(x[ib].dm);
+
+ const uint8_t * q = x[ib].qs + 32*il + n*ir;
+
+ uint8_t sc, m;
+ get_scale_min_k4(is + 0, x[ib].scales, sc, m);
+ const float d1 = dall * sc; const float m1 = dmin * m;
+ get_scale_min_k4(is + 1, x[ib].scales, sc, m);
+ const float d2 = dall * sc; const float m2 = dmin * m;
+ for (int l = 0; l < n; ++l) {
+ y[l + 0] = ggml_cuda_cast<dst_t>(d1 * (q[l] & 0xF) - m1);
+ y[l +32] = ggml_cuda_cast<dst_t>(d2 * (q[l] >> 4) - m2);
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_q5_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {
+ const block_q5_K * x = (const block_q5_K *) vx;
+
+ // assume 64 threads - this is very slightly better than the one below
+ const int64_t il = tid/16; // il is in 0...3
+ const int64_t ir = tid%16; // ir is in 0...15
+ const int64_t is = 2*il; // is is in 0...6
+
+ dst_t * y = yy + 64*il + 2*ir;
+
+ const float dall = __low2half(x[ib].dm);
+ const float dmin = __high2half(x[ib].dm);
+
+ const uint8_t * ql = x[ib].qs + 32*il + 2*ir;
+ const uint8_t * qh = x[ib].qh + 2*ir;
+
+ uint8_t sc, m;
+ get_scale_min_k4(is + 0, x[ib].scales, sc, m);
+ const float d1 = dall * sc; const float m1 = dmin * m;
+ get_scale_min_k4(is + 1, x[ib].scales, sc, m);
+ const float d2 = dall * sc; const float m2 = dmin * m;
+
+ uint8_t hm = 1 << (2*il);
+ y[ 0] = ggml_cuda_cast<dst_t>(d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1);
+ y[ 1] = ggml_cuda_cast<dst_t>(d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1);
+ hm <<= 1;
+ y[32] = ggml_cuda_cast<dst_t>(d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2);
+ y[33] = ggml_cuda_cast<dst_t>(d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2);
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_q6_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {
+ const block_q6_K * x = (const block_q6_K *) vx;
+
+ // assume 64 threads - this is very slightly better than the one below
+ const int64_t ip = tid/32; // ip is 0 or 1
+ const int64_t il = tid - 32*ip; // 0...32
+ const int64_t is = 8*ip + il/16;
+
+ dst_t * y = yy + 128*ip + il;
+
+ const float d = x[ib].d;
+
+ const uint8_t * ql = x[ib].ql + 64*ip + il;
+ const uint8_t qh = x[ib].qh[32*ip + il];
+ const int8_t * sc = x[ib].scales + is;
+
+ y[ 0] = ggml_cuda_cast<dst_t>(d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32));
+ y[32] = ggml_cuda_cast<dst_t>(d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32));
+ y[64] = ggml_cuda_cast<dst_t>(d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32));
+ y[96] = ggml_cuda_cast<dst_t>(d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32));
+}
+
+//================================== i-quants
+
+// Each call dequantizes one super-block of QK_K values into y with 32
+// threads; iq4_nl packs QK_K/QK4_NL sub-blocks per super-block.
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq2_xxs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_iq2_xxs * x = (const block_iq2_xxs *) vx;
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 8*il;
+ const uint16_t * q2 = x[ibs].qs + 4*ib;
+ const uint8_t * aux8 = (const uint8_t *)q2;
+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);
+ const uint32_t aux32 = q2[2] | (q2[3] << 16);
+ const float d = (float)x[ibs].d * (0.5f + (aux32 >> 28)) * 0.25f;
+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
+ for (int j = 0; j < 8; ++j) {
+ y[j] = ggml_cuda_cast<dst_t>(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq2_xs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_iq2_xs * x = (const block_iq2_xs *) vx;
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 8*il;
+ const uint16_t * q2 = x[ibs].qs + 4*ib;
+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));
+ const float d = (float)x[ibs].d * (0.5f + ((x[ibs].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
+ const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];
+ for (int j = 0; j < 8; ++j) {
+ y[j] = ggml_cuda_cast<dst_t>(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq2_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_iq2_s * x = (const block_iq2_s *) vx;
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 8*il;
+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[ibs].qs[4*ib+il] | ((x[ibs].qh[ib] << (8-2*il)) & 0x300)));
+ const float d = (float)x[ibs].d * (0.5f + ((x[ibs].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
+ const uint8_t signs = x[ibs].qs[QK_K/8+4*ib+il];
+ for (int j = 0; j < 8; ++j) {
+ y[j] = ggml_cuda_cast<dst_t>(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq3_xxs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_iq3_xxs * x = (const block_iq3_xxs *) vx;
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 8*il;
+ const uint8_t * q3 = x[ibs].qs + 8*ib;
+ const uint16_t * gas = (const uint16_t *)(x[ibs].qs + QK_K/4) + 2*ib;
+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);
+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);
+ const uint32_t aux32 = gas[0] | (gas[1] << 16);
+ const float d = (float)x[ibs].d * (0.5f + (aux32 >> 28)) * 0.5f;
+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
+ for (int j = 0; j < 4; ++j) {
+ y[j+0] = ggml_cuda_cast<dst_t>(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f));
+ y[j+4] = ggml_cuda_cast<dst_t>(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f));
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq3_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_iq3_s * x = (const block_iq3_s *) vx;
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 8*il;
+ const uint8_t * qs = x[ibs].qs + 8*ib;
+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[ibs].qh[ib] << (8-2*il)) & 256)));
+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[ibs].qh[ib] << (7-2*il)) & 256)));
+ const float d = (float)x[ibs].d * (1 + 2*((x[ibs].scales[ib/2] >> 4*(ib%2)) & 0xf));
+ const uint8_t signs = x[ibs].signs[4*ib + il];
+ for (int j = 0; j < 4; ++j) {
+ y[j+0] = ggml_cuda_cast<dst_t>(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f));
+ y[j+4] = ggml_cuda_cast<dst_t>(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f));
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq1_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_iq1_s * x = (const block_iq1_s *) vx;
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 8*il;
+ const float delta = x[ibs].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;
+ const float d = (float)x[ibs].d * (2*((x[ibs].qh[ib] >> 12) & 7) + 1);
+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
+ grid32[0] = iq1s_grid_gpu[x[ibs].qs[4*ib+il] | (((x[ibs].qh[ib] >> 3*il) & 7) << 8)];
+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
+ grid32[0] &= 0x0f0f0f0f;
+ for (int j = 0; j < 8; ++j) {
+ y[j] = ggml_cuda_cast<dst_t>(d * (q[j] + delta));
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq1_m(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_iq1_m * x = (const block_iq1_m *) vx;
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 8*il;
+ const uint16_t * sc = (const uint16_t *)x[ibs].scales;
+ iq1m_scale_t scale;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
+ const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);
+ const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);
+ const float delta = x[ibs].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;
+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
+ grid32[0] = iq1s_grid_gpu[x[ibs].qs[4*ib+il] | (((x[ibs].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];
+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
+ grid32[0] &= 0x0f0f0f0f;
+ for (int j = 0; j < 8; ++j) {
+ y[j] = ggml_cuda_cast<dst_t>(d * (q[j] + delta));
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq4_nl(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_iq4_nl * x = (const block_iq4_nl *) vx + ibs*(QK_K/QK4_NL);
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 4*il;
+ const uint8_t * q4 = x[ib].qs + 4*il;
+ const float d = (float)x[ib].d;
+ for (int j = 0; j < 4; ++j) {
+ y[j+ 0] = ggml_cuda_cast<dst_t>(d * kvalues_iq4nl[q4[j] & 0xf]);
+ y[j+16] = ggml_cuda_cast<dst_t>(d * kvalues_iq4nl[q4[j] >> 4]);
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_iq4_xs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+ const block_iq4_xs * x = (const block_iq4_xs *)vx;
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 4*il;
+ const uint8_t * q4 = x[ibs].qs + 16*ib + 4*il;
+ const float d = (float)x[ibs].d * ((((x[ibs].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[ibs].scales_h >> 2*ib) & 3) << 4)) - 32);
+ for (int j = 0; j < 4; ++j) {
+ y[j+ 0] = ggml_cuda_cast<dst_t>(d * kvalues_iq4nl[q4[j] & 0xf]);
+ y[j+16] = ggml_cuda_cast<dst_t>(d * kvalues_iq4nl[q4[j] >> 4]);
+ }
+}
+
+template<typename dst_t>
+static __device__ __forceinline__ void dequantize_mxfp4(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {
+
+ const block_mxfp4 * x = (const block_mxfp4 *) vx + ibs*(QK_K/QK_MXFP4);
+
+ const int64_t il = tid/8; // 0...3
+ const int64_t ib = tid%8; // 0...7
+ dst_t * y = yy + 32*ib + 4*il;
+ const uint8_t * q4 = x[ib].qs + 4*il;
+ const float d = ggml_cuda_e8m0_to_fp32(x[ib].e);
+ for (int j = 0; j < 4; ++j) {
+ y[j+ 0] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q4[j] & 0xf]*0.5f);
+ y[j+16] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q4[j] >> 4]*0.5f);
+ }
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cu
new file mode 100644
index 00000000..d97e8fdb
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cu
@@ -0,0 +1,488 @@
+#include "getrows.cuh"
+#include "dequantize.cuh"
+#include "convert.cuh"
+
+template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
+static __global__ void k_get_rows(
+ const void * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst,
+ const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/
+ /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/
+ /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3,
+ /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03,
+ const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) {
+
+ ggml_cuda_pdl_sync();
+ for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) {
+ for (int64_t i00 = 2*(blockIdx.y*blockDim.x + threadIdx.x); i00 < ne00; i00 += gridDim.y*blockDim.x) {
+ // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher.
+ const int i10 = blockIdx.x;
+ const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv);
+ const int i11 = dm.x;
+ const int i12 = dm.y;
+
+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
+
+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
+ const void * src0_row = (const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03;
+
+ const int ib = i00/qk; // block index
+ const int iqs = (i00%qk)/qr; // quant index
+ const int iybs = i00 - i00%qk; // dst block start index
+ const int y_offset = qr == 1 ? 1 : qk/2;
+
+ // dequantize
+ float2 v;
+ dequantize_kernel(src0_row, ib, iqs, v);
+
+ dst_row[iybs + iqs + 0] = ggml_cuda_cast<dst_t>(v.x);
+ dst_row[iybs + iqs + y_offset] = ggml_cuda_cast<dst_t>(v.y);
+ }
+ }
+}
+
+template<typename dst_t, dequantize_kq_t<dst_t> dequantize_kq>
+static __global__ void k_get_rows_kq(
+ const void * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst,
+ const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/
+ /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/
+ /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3,
+ /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03,
+ const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) {
+
+ ggml_cuda_pdl_sync();
+ const int64_t nsb = ne00/QK_K; // super-blocks per row
+ for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) {
+ // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher.
+ const int i10 = blockIdx.x;
+ const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv);
+ const int i11 = dm.x;
+ const int i12 = dm.y;
+
+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
+
+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
+ const void * src0_row = (const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03;
+
+ for (int64_t ib = blockIdx.y; ib < nsb; ib += gridDim.y) {
+ dequantize_kq(src0_row, ib, dst_row + ib*QK_K, threadIdx.x);
+ }
+ }
+}
+
+template<typename src0_t, typename dst_t>
+static __global__ void k_get_rows_float(
+ const src0_t * src0_ptr, const int32_t * src1_ptr, dst_t * dst_ptr,
+ const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/
+ /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/
+ /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3,
+ /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03,
+ const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) {
+
+ ggml_cuda_pdl_lc();
+ const src0_t * GGML_CUDA_RESTRICT src0 = src0_ptr;
+ const int32_t * GGML_CUDA_RESTRICT src1 = src1_ptr;
+ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;
+ ggml_cuda_pdl_sync();
+ for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) {
+ // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher.
+ const int i10 = blockIdx.x;
+ const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv);
+ const int i11 = dm.x;
+ const int i12 = dm.y;
+
+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
+
+ dst_t * GGML_CUDA_RESTRICT dst_row = dst + i10*s1 + i11*s2 + i12*s3;
+ const src0_t * GGML_CUDA_RESTRICT src0_row = (const src0_t *)((const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03);
+
+ for (int64_t i00 = blockIdx.y*blockDim.x + threadIdx.x; i00 < ne00; i00 += gridDim.y*blockDim.x) {
+ dst_row[i00] = ggml_cuda_cast<dst_t>(src0_row[i00]);
+ }
+ }
+}
+
+template<typename dst_t>
+static __global__ void k_get_rows_float_vec(
+ const dst_t * src0_ptr, const int32_t * src1_ptr, dst_t * dst_ptr,
+ const int64_t ne00v,
+ const int64_t ne11, const uint3 ne12_fdv,
+ const size_t s1, const size_t s2, const size_t s3,
+ const size_t nb01, const size_t nb02, const size_t nb03,
+ const size_t s10, const size_t s11, const size_t s12) {
+
+ ggml_cuda_pdl_lc();
+ ggml_cuda_pdl_sync();
+ for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) {
+ const int i10 = blockIdx.x;
+ const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv);
+ const int i11 = dm.x;
+ const int i12 = dm.y;
+
+ const int i01 = src1_ptr[i10*s10 + i11*s11 + i12*s12];
+
+ int4 * GGML_CUDA_RESTRICT dst_row = (int4 *) (dst_ptr + i10*s1 + i11*s2 + i12*s3);
+ const int4 * GGML_CUDA_RESTRICT src0_row = (const int4 *)((const char *) src0_ptr + i01*nb01 + i11*nb02 + i12*nb03);
+
+ for (int64_t i = blockIdx.y*blockDim.x + threadIdx.x; i < ne00v; i += gridDim.y*blockDim.x) {
+ dst_row[i] = src0_row[i];
+ }
+ }
+}
+
+template<typename grad_t, typename dst_t>
+static __global__ void k_get_rows_back_float(
+ const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst,
+ const int64_t ncols, const int64_t nrows_grad, const int64_t nrows_dst) {
+ const int col = blockIdx.x*blockDim.x + threadIdx.x;
+
+ if (col >= ncols) {
+ return;
+ }
+
+ ggml_cuda_pdl_sync();
+
+ // grid.y is clamped to the CUDA grid limit, so stride over the destination rows
+ for (int64_t dst_row = blockIdx.y; dst_row < nrows_dst; dst_row += gridDim.y) {
+ float sum = 0.0f;
+
+ for (int64_t i = 0; i < nrows_grad; ++i) {
+ if (rows[i] != dst_row) {
+ continue;
+ }
+ sum += grad[i*ncols + col];
+ }
+
+ dst[dst_row*ncols + col] = sum;
+ }
+}
+
+template<int qk, int qr, dequantize_kernel_t dq, typename dst_t>
+static void get_rows_cuda_q(
+ const void * src0_d, const int32_t * src1_d, dst_t * dst_d,
+ const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,
+ const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,
+ const size_t nb1, const size_t nb2, const size_t nb3,
+ cudaStream_t stream) {
+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);
+ const int block_num_y = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);
+ const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));
+
+ // strides in elements
+ // const size_t s0 = nb0 / sizeof(dst_t);
+ const size_t s1 = nb1 / sizeof(dst_t);
+ const size_t s2 = nb2 / sizeof(dst_t);
+ const size_t s3 = nb3 / sizeof(dst_t);
+
+ const size_t s10 = nb10 / sizeof(int32_t);
+ const size_t s11 = nb11 / sizeof(int32_t);
+ const size_t s12 = nb12 / sizeof(int32_t);
+ // const size_t s13 = nb13 / sizeof(int32_t);
+
+ GGML_ASSERT(ne00 % 2 == 0);
+
+ GGML_ASSERT(ne12 > 0);
+ GGML_ASSERT(ne11 <= std::numeric_limits<uint32_t>::max() / ne12);
+ const uint3 ne12_fdv = init_fastdiv_values(ne12);
+
+ k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(
+ src0_d, src1_d, dst_d,
+ ne00, /*ne01, ne02, ne03,*/
+ /*ne10,*/ ne11, ne12_fdv, /*ne13,*/
+ /* s0,*/ s1, s2, s3,
+ /* nb00,*/ nb01, nb02, nb03,
+ s10, s11, s12/*, s13*/);
+}
+
+template<int block_dim, typename dst_t, dequantize_kq_t<dst_t> dequantize_kq>
+static void get_rows_cuda_kq(
+ const void * src0_d, const int32_t * src1_d, dst_t * dst_d,
+ const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,
+ const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,
+ const size_t nb1, const size_t nb2, const size_t nb3,
+ cudaStream_t stream) {
+ GGML_ASSERT(ne00 % QK_K == 0);
+ const int64_t nsb = ne00/QK_K;
+
+ const dim3 block_dims(block_dim, 1, 1);
+ const dim3 block_nums(ne10, MIN(nsb, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));
+
+ // strides in elements
+ // const size_t s0 = nb0 / sizeof(dst_t);
+ const size_t s1 = nb1 / sizeof(dst_t);
+ const size_t s2 = nb2 / sizeof(dst_t);
+ const size_t s3 = nb3 / sizeof(dst_t);
+
+ const size_t s10 = nb10 / sizeof(int32_t);
+ const size_t s11 = nb11 / sizeof(int32_t);
+ const size_t s12 = nb12 / sizeof(int32_t);
+ // const size_t s13 = nb13 / sizeof(int32_t);
+
+ GGML_ASSERT(ne12 > 0);
+ GGML_ASSERT(ne11 <= std::numeric_limits<uint32_t>::max() / ne12);
+ const uint3 ne12_fdv = init_fastdiv_values(ne12);
+
+ k_get_rows_kq<dst_t, dequantize_kq><<<block_nums, block_dims, 0, stream>>>(
+ src0_d, src1_d, dst_d,
+ ne00, /*ne01, ne02, ne03,*/
+ /*ne10,*/ ne11, ne12_fdv, /*ne13,*/
+ /* s0,*/ s1, s2, s3,
+ /* nb00,*/ nb01, nb02, nb03,
+ s10, s11, s12/*, s13*/);
+}
+
+template<typename src0_t, typename dst_t>
+static void get_rows_cuda_float(
+ const src0_t * src0_d, const int32_t * src1_d, dst_t * dst_d,
+ const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,
+ const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,
+ const size_t nb1, const size_t nb2, const size_t nb3,
+ cudaStream_t stream) {
+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);
+
+ // strides in elements
+ // const size_t s0 = nb0 / sizeof(dst_t);
+ const size_t s1 = nb1 / sizeof(dst_t);
+ const size_t s2 = nb2 / sizeof(dst_t);
+ const size_t s3 = nb3 / sizeof(dst_t);
+
+ const size_t s10 = nb10 / sizeof(int32_t);
+ const size_t s11 = nb11 / sizeof(int32_t);
+ const size_t s12 = nb12 / sizeof(int32_t);
+ // const size_t s13 = nb13 / sizeof(int32_t);
+
+ GGML_ASSERT(ne12 > 0);
+ GGML_ASSERT(ne11 <= std::numeric_limits<uint32_t>::max() / ne12);
+ const uint3 ne12_fdv = init_fastdiv_values(ne12);
+
+ if constexpr (std::is_same<src0_t, dst_t>::value) {
+ constexpr int VEC = 16 / sizeof(dst_t);
+ const int64_t ne00v = ne00 / VEC;
+ const int64_t vec_block_num_y = (ne00v + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;
+ const bool enough_blocks = vec_block_num_y * ne10 * ne11 * ne12 >= 128;
+ const bool can_vec = VEC > 1 && enough_blocks &&
+ (ne00 % VEC == 0) &&
+ (nb01 % 16 == 0) && (nb02 % 16 == 0) && (nb03 % 16 == 0) &&
+ (nb1 % 16 == 0) && (nb2 % 16 == 0) && (nb3 % 16 == 0) &&
+ (((uintptr_t) src0_d) % 16 == 0) && (((uintptr_t) dst_d) % 16 == 0);
+
+ if (can_vec) {
+ const int block_num_y = vec_block_num_y;
+ const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{block_nums, block_dims, 0, stream};
+ ggml_cuda_kernel_launch(k_get_rows_float_vec<dst_t>, launch_params,
+ (const dst_t *) src0_d, src1_d, dst_d,
+ ne00v, ne11, ne12_fdv,
+ s1, s2, s3,
+ nb01, nb02, nb03,
+ s10, s11, s12);
+ return;
+ }
+ }
+
+ const int block_num_y = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;
+ const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));
+
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{block_nums, block_dims, 0, stream};
+ ggml_cuda_kernel_launch(k_get_rows_float<src0_t, dst_t>, launch_params,
+ src0_d, src1_d, dst_d,
+ ne00, /*ne01, ne02, ne03,*/
+ /*ne10,*/ ne11, ne12_fdv, /*ne13,*/
+ /* s0,*/ s1, s2, s3,
+ /* nb00,*/ nb01, nb02, nb03,
+ s10, s11, s12/*, s13*/);
+}
+
+template <typename dst_t>
+static void ggml_cuda_get_rows_switch_src0_type(
+ const void * src0_d, const ggml_type src0_type, const int32_t * src1_d, dst_t * dst_d,
+ const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,
+ const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,
+ const size_t nb1, const size_t nb2, const size_t nb3,
+ cudaStream_t stream) {
+ switch (src0_type) {
+ case GGML_TYPE_F16:
+ get_rows_cuda_float((const half *) src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_F32:
+ get_rows_cuda_float((const float *) src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_I32:
+ get_rows_cuda_float((const int32_t *) src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_BF16:
+ get_rows_cuda_float((const nv_bfloat16 *) src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q1_0:
+ get_rows_cuda_q<QK1_0, QR1_0, dequantize_q1_0>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q2_0:
+ get_rows_cuda_q<QK2_0, QR2_0, dequantize_q2_0>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q4_0:
+ get_rows_cuda_q<QK4_0, QR4_0, dequantize_q4_0>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q4_1:
+ get_rows_cuda_q<QK4_1, QR4_1, dequantize_q4_1>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q5_0:
+ get_rows_cuda_q<QK5_0, QR5_0, dequantize_q5_0>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q5_1:
+ get_rows_cuda_q<QK5_1, QR5_1, dequantize_q5_1>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q8_0:
+ get_rows_cuda_q<QK8_0, QR8_0, dequantize_q8_0>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q2_K:
+ get_rows_cuda_kq<64, dst_t, dequantize_q2_K<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q3_K:
+ get_rows_cuda_kq<64, dst_t, dequantize_q3_K<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q4_K:
+ get_rows_cuda_kq<32, dst_t, dequantize_q4_K<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q5_K:
+ get_rows_cuda_kq<64, dst_t, dequantize_q5_K<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_Q6_K:
+ get_rows_cuda_kq<64, dst_t, dequantize_q6_K<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ2_XXS:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq2_xxs<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ2_XS:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq2_xs<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ2_S:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq2_s<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ3_XXS:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq3_xxs<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ3_S:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq3_s<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ1_S:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq1_s<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ1_M:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq1_m<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ4_NL:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq4_nl<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_IQ4_XS:
+ get_rows_cuda_kq<32, dst_t, dequantize_iq4_xs<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_MXFP4:
+ get_rows_cuda_kq<32, dst_t, dequantize_mxfp4<dst_t>>(src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ default:
+ GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0_type));
+ }
+}
+
+void get_rows_cuda(
+ const void * src0_d, ggml_type src0_type, const int32_t * src1_d, void * dst_d, ggml_type dst_type,
+ int64_t ne00, size_t nb01, size_t nb02, size_t nb03,
+ int64_t ne10, int64_t ne11, int64_t ne12, size_t nb10, size_t nb11, size_t nb12,
+ size_t nb1, size_t nb2, size_t nb3,
+ cudaStream_t stream) {
+ switch (dst_type) {
+ case GGML_TYPE_F32:
+ ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (float *) dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_I32:
+ ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (int32_t *) dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_F16:
+ ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (half *) dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ case GGML_TYPE_BF16:
+ ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (nv_bfloat16 *) dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
+ default:
+ GGML_ABORT("%s: unsupported dst type: %s\n", __func__, ggml_type_name(dst_type));
+ }
+}
+
+void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+
+ cudaStream_t stream = ctx.stream();
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(src1->type == GGML_TYPE_I32);
+ GGML_ASSERT(ne13 == 1);
+
+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
+ GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));
+ GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));
+
+ get_rows_cuda(src0->data, src0->type, (const int32_t *) src1->data, dst->data, dst->type,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+}
+
+void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0]; // gradients of forward pass output
+ const ggml_tensor * src1 = dst->src[1]; // src1 in forward pass
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const float * src0_d = (const float *) src0->data;
+ const int32_t * src1_d = (const int32_t *) src1->data;
+ float * dst_d = (float *) dst->data;
+
+ cudaStream_t stream = ctx.stream();
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT(src1->type == GGML_TYPE_I32);
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+
+ GGML_ASSERT(ggml_is_contiguous(src0));
+ GGML_ASSERT(ggml_is_contiguous(src1));
+ GGML_ASSERT(ggml_is_contiguous(dst));
+
+ GGML_ASSERT(ne02*ne03 == 1);
+ GGML_ASSERT(ne12*ne13 == 1);
+ GGML_ASSERT(ne2*ne3 == 1);
+
+ const dim3 block_dims(CUDA_GET_ROWS_BACK_BLOCK_SIZE, 1, 1);
+ const int block_num_x = (ne00 + CUDA_GET_ROWS_BACK_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BACK_BLOCK_SIZE;
+ const dim3 block_nums(block_num_x, MIN(ne1, (int64_t)UINT16_MAX), 1);
+
+ k_get_rows_back_float<<<block_nums, block_dims, 0, stream>>>(src0_d, src1_d, dst_d, ne00, ne10, ne1);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cuh
new file mode 100644
index 00000000..3c5bea5f
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cuh
@@ -0,0 +1,15 @@
+#include "common.cuh"
+
+#define CUDA_GET_ROWS_BLOCK_SIZE 256
+#define CUDA_GET_ROWS_BACK_BLOCK_SIZE 256
+
+void get_rows_cuda(
+ const void * src0_d, ggml_type src0_type, const int32_t * src1_d, void * dst_d, ggml_type dst_type,
+ int64_t ne00, size_t nb01, size_t nb02, size_t nb03,
+ int64_t ne10, int64_t ne11, int64_t ne12, size_t nb10, size_t nb11, size_t nb12,
+ size_t nb1, size_t nb2, size_t nb3,
+ cudaStream_t stream);
+
+void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mma.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mma.cuh
new file mode 100644
index 00000000..93a51aa0
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mma.cuh
@@ -0,0 +1,1594 @@
+#pragma once
+// This file contains primitives that expose the tensor core PTX instructions for CUDA code.
+// The primitives can be used in a similar way as the nvcuda::wmma interface but with a well-defined memory layout.
+// The documentation for the PTX instructions can be found under:
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-multiply-accumulate-operation-using-mma-instruction
+//
+// Like with nvcuda::wmma there are three types of matrix tiles: A, B, and C with A @ B = C.
+// A is a row-major matrix with shape M x K.
+// B is a column-major matrix with shape K x N.
+// C is a column-major matrix with shape M x N.
+// A, B, and C are represented using the same fundamental data type: a row-major matrix with I rows and J columns.
+// Note that J is measured in physical 32 bit elements instead of logical elements.
+// The methods get_i and get_j can be used to get the physical 32 bit index of the lth element of a thread within a tile.
+// All matrix tiles have ne physical 32 bit elements per warp.
+//
+// As described in the PTX documentation, all pointers for load_ldmatrix must be to shared memory and aligned to 16 bytes.
+// The API in this file also assumes that the pointers for load_generic are aligned to 16 bytes, unaligned pointers are considered undefined behavior.
+
+#include "common.cuh"
+
+// On Volta each warp is doing 4 8x8 mma operations in parallel.
+// The basic memory layout for a 32x8 output tile is to stack 4 input tiles in I direction and to mirror the B tile.
+// However, the i indices in this file are by default permuted to simplify the index calculations.
+// #define GGML_CUDA_MMA_NO_VOLTA_PERM
+
+#if CUDART_VERSION >= 11080
+
+static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {
+ int ret = 0;
+
+#ifdef TURING_MMA_AVAILABLE
+ asm("movmatrix.sync.aligned.m8n8.trans.b16 %0, %1;"
+ : "=r"(ret) : "r"(x));
+#else
+ GGML_UNUSED(x);
+ NO_DEVICE_CODE;
+#endif // defined(TURING_MMA_AVAILABLE)
+ return ret;
+}
+
+#else
+
+static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {
+ // Imagine transposing row-major matrix to column-major matrix.
+ const int src_i_low = 2 * (threadIdx.x % 4);
+ const int src_i_high = src_i_low + 1;
+ const int src_j = threadIdx.x / 4;
+
+ const int src_laneid_low = src_i_low * 4 + src_j / 2;
+ const int src_laneid_high = src_i_high * 4 + src_j / 2;
+
+ const int shift_low = ((src_j + 0) % 2) * 16;
+ const int shift_high = ((src_j + 1) % 2) * 16;
+
+ const int ret_low = (__shfl_sync(0xFFFFFFFF, x, src_laneid_low, WARP_SIZE) >> shift_low) & 0x0000FFFF;
+ const int ret_high = (__shfl_sync(0xFFFFFFFF, x, src_laneid_high, WARP_SIZE) << shift_high) & 0xFFFF0000;
+
+ return ret_low | ret_high;
+}
+
+#endif // CUDART_VERSION >= 11080
+
+static __device__ __forceinline__ half2 ggml_cuda_movmatrix(const half2 x) {
+ half2 ret;
+ *((int *) &ret) = ggml_cuda_movmatrix(*((const int *) &x));
+ return ret;
+}
+
+namespace ggml_cuda_mma {
+
+ // Some architectures like Volta or CDNA3 perform multiple matrix multiplications per warp in parallel,
+ // effectively the warp is being split into subgroups of threads that each perform a single mma instruction.
+ // In those cases the data can be split in different ways across the warp.
+ enum data_layout {
+ // By default the data uses the I direction as its major dimension and the J direction as its minor dimension.
+ // For the A/C matrices this means I major == row major, J major == column major.
+ // For the B matrix this means I major == column major, J major == row major.
+ // MIRRORED == Each data value is held exactly once per thread subgroup.
+ DATA_LAYOUT_I_MAJOR = 0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for RDNA4 and CDNA.
+ DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3.
+ DATA_LAYOUT_I_MAJOR_MIRRORED = 20, // Volta, matrix A&B for RDNA3.
+ DATA_LAYOUT_J_MAJOR_MIRRORED = 30,
+ DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (RDNA4/CDNA), convert to float to unscramble.
+ };
+ // Implemented mma combinations are:
+ // - (I_MAJOR, I_MAJOR) -> I_MAJOR
+ // - (I_MAJOR, I_MAJOR_MIRRORED) -> I_MAJOR
+ // - (I_MAJOR, J_MAJOR_MIRRORED) -> I_MAJOR
+
+ static constexpr __device__ data_layout get_input_data_layout() {
+#if defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE)
+ return DATA_LAYOUT_I_MAJOR_MIRRORED;
+#else
+ return DATA_LAYOUT_I_MAJOR;
+#endif // defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE)
+ }
+
+ template <int I_, int J_, typename T, data_layout ds_=DATA_LAYOUT_I_MAJOR>
+ struct tile {};
+
+ template <int I_, int J_, typename T>
+ struct tile<I_, J_, T, DATA_LAYOUT_I_MAJOR> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;
+
+#if defined(AMD_MFMA_AVAILABLE)
+ static constexpr int ne = I * J / 64;
+ T x[ne] = {0};
+
+ static constexpr __device__ bool supported() {
+ if (I == 16 && J == 8) return true;
+ if (I == 32 && J == 4) return true;
+ if (I == 16 && J == 16) return true;
+ if (I == 32 && J == 32) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 16 && J == 4) {
+ return threadIdx.x % 16;
+ } else if constexpr (I == 16 && J == 8) {
+ return threadIdx.x % 16;
+ } else if constexpr (I == 32 && J == 4) {
+ return threadIdx.x % 32;
+ } else if constexpr (I == 16 && J == 16) {
+ return threadIdx.x % 16;
+ } else if constexpr (I == 32 && J == 32) {
+ return threadIdx.x % 32;
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 16 && J == 4) {
+ return threadIdx.x / 16;
+ } else if constexpr (I == 16 && J == 8) {
+ return 2 * (threadIdx.x / 16) + l;
+ } else if constexpr (I == 32 && J == 4) {
+ return 2 * (threadIdx.x / 32) + l;
+ } else if constexpr (I == 16 && J == 16) {
+ return 4 * (threadIdx.x / 16) + l;
+ } else if constexpr (I == 32 && J == 32) {
+ return 4 * (threadIdx.x / 32) + 8 * (l / 4) + (l % 4);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#elif defined(VOLTA_MMA_AVAILABLE)
+ static constexpr int ne = I * J / 32;
+ T x[ne] = {0};
+
+ static constexpr __device__ bool supported() {
+ if (I == 32 && J == 8) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 32 && J == 8) {
+#ifdef GGML_CUDA_MMA_NO_VOLTA_PERM
+ return (((threadIdx.x % 16) / 4) * 8) + ((threadIdx.x / 16) * 4) + (l & 2) + (threadIdx.x % 2);
+#else
+ return (l & 2) + (threadIdx.x & ~2);
+#endif // GGML_CUDA_MMA_NO_VOLTA_PERM
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 32 && J == 8) {
+ return (threadIdx.x & 2) + (l & (4 + 1));
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#elif defined(AMD_WMMA_AVAILABLE)
+ static constexpr int ne = I * J / 32;
+ T x[ne] = {0};
+
+ static constexpr __device__ bool supported() {
+ if (I == 16 && J == 16) return true;
+ if (I == 16 && J == 8) return true;
+ if (I == 16 && J == 4) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (supported()) {
+ return threadIdx.x % 16;
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 16 && J == 16) {
+#if defined(RDNA3)
+ if constexpr (std::is_same_v<T, float> || std::is_same_v<T, int>) {
+ // matrix C
+ return 2 * l + (threadIdx.x / 16);
+ } else {
+ // matrix A&B
+ return l;
+ }
+#else
+ // matrix C is the transposed matrix A&B on RDNA4
+ return ne * (threadIdx.x / 16) + l;
+#endif // defined(RDNA3)
+ } else if constexpr (I == 16 && J == 8) {
+ // mmq input for RDNA4
+ return ne * (threadIdx.x / 16) + l;
+ } else if constexpr (I == 16 && J == 4) {
+ return ne * (threadIdx.x / 16) + l;
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#else
+ static constexpr int ne = I * J / 32;
+ T x[ne] = {0};
+
+ static constexpr __device__ bool supported() {
+ if (I == 8 && J == 4) return true;
+ if (I == 8 && J == 8) return true;
+ if (I == 16 && J == 8) return true;
+ if (I == 16 && J == 16) return true;
+ if (I == 32 && J == 8) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 8 && J == 4) {
+ return threadIdx.x / 4;
+ } else if constexpr (I == 8 && J == 8) {
+ return threadIdx.x / 4;
+ } else if constexpr (I == 16 && J == 8) {
+ return ((l / 2) * 8) + (threadIdx.x / 4);
+ } else if constexpr (I == 16 && J == 16) {
+ return (((l / 2) % 2) * 8) + (threadIdx.x / 4);
+ } else if constexpr (I == 32 && J == 8) {
+ return tile<16, 8, T>::get_i(l); // Memory layout simply repeated with same pattern in i direction.
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 8 && J == 4) {
+ return threadIdx.x % 4;
+ } else if constexpr (I == 8 && J == 8) {
+ return (l * 4) + (threadIdx.x % 4);
+ } else if constexpr (I == 16 && J == 8) {
+ return ((threadIdx.x % 4) * 2) + (l % 2);
+ } else if constexpr (I == 16 && J == 16) {
+ return ((l / 4) * 8) + ((threadIdx.x % 4) * 2) + (l % 2);
+ } else if constexpr (I == 32 && J == 8) {
+ return tile<16, 8, T>::get_j(l); // Memory layout simply repeated with same pattern in i direction.
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#endif // defined(GGML_USE_HIP)
+ };
+
+ template <int I_, int J_>
+ struct tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;
+
+#if defined(VOLTA_MMA_AVAILABLE)
+ static constexpr int ne = I * J / WARP_SIZE;
+ half2 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ if (I == 32 && J == 4) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 32 && J == 4) {
+#ifdef GGML_CUDA_MMA_NO_VOLTA_PERM
+ return (((threadIdx.x % 16) / 4) * 8) + ((threadIdx.x / 16) * 4) + (threadIdx.x % 4);
+#else
+ return threadIdx.x;
+#endif // GGML_CUDA_MMA_NO_VOLTA_PERM
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 32 && J == 4) {
+ return l;
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#elif defined(AMD_WMMA_AVAILABLE)
+ static constexpr int ne = I * J / 32;
+ half2 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ if (I == 16 && J == 8) return true;
+ if (I == 16 && J == 16) return true;
+ if (I == 32 && J == 8) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 16 && J == 8) {
+ return threadIdx.x % 16;
+ } else if constexpr (I == 16 && J == 16) {
+ return threadIdx.x % 16;
+ } else if constexpr (I == 32 && J == 8) {
+ return (threadIdx.x % 16) * 2 + l / (ne/2);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 16 && J == 8) {
+ return (threadIdx.x / 16) * ne + l;
+ } else if constexpr (I == 16 && J == 16) {
+#ifdef RDNA3
+ return l*2 + (threadIdx.x / 16);
+#else
+ return (threadIdx.x / 16) * ne + l;
+#endif // RDNA3
+ } else if constexpr (I == 32 && J == 8) {
+ return (threadIdx.x / 16) * (ne/2) + l % (ne/2);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#elif defined(AMD_MFMA_AVAILABLE)
+ static constexpr int ne = I * J / 64;
+ half2 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ if (I == 16 && J == 8) return true;
+ if (I == 16 && J == 16) return true;
+ if (I == 32 && J == 8) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 16 && J == 8) {
+ return threadIdx.x % 16;
+ } else if constexpr (I == 16 && J == 16) {
+ return threadIdx.x % 16;
+ } else if constexpr (I == 32 && J == 8) {
+ return (threadIdx.x % 16) * 2 + l / (ne/2);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 16 && J == 8) {
+ return (threadIdx.x / 16) * ne + l;
+ } else if constexpr (I == 16 && J == 16) {
+ return (threadIdx.x / 16) * ne + l;
+ } else if constexpr (I == 32 && J == 8) {
+ return (threadIdx.x / 16) * (ne/2) + l % (ne/2);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#else
+ static constexpr int ne = I * J / WARP_SIZE;
+ half2 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ if (I == 8 && J == 4) return true;
+ if (I == 8 && J == 8) return true;
+ if (I == 16 && J == 8) return true;
+ if (I == 16 && J == 16) return true;
+ if (I == 32 && J == 8) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 8 && J == 8) {
+ return threadIdx.x / 4;
+ } else if constexpr (I == 16 && J == 4) {
+ return (l * 8) + (threadIdx.x / 4);
+ } else if constexpr (I == 16 && J == 8) {
+ return ((l % 2) * 8) + (threadIdx.x / 4);
+ } else if constexpr (I == 32 && J == 8) {
+ return ((l / 4) * 16) + ((l % 2) * 8) + (threadIdx.x / 4);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 8 && J == 8) {
+ return (l * 4) + (threadIdx.x % 4);
+ } else if constexpr (I == 16 && J == 4) {
+ return threadIdx.x % 4;
+ } else if constexpr (I == 16 && J == 8) {
+ return ((l / 2) * 4) + (threadIdx.x % 4);
+ } else if constexpr (I == 32 && J == 8) {
+ return ((l & 2) * 2) + (threadIdx.x % 4);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#endif // defined(VOLTA_MMA_AVAILABLE)
+ };
+
+ template <int I_, int J_>
+ struct tile<I_, J_, nv_bfloat162, DATA_LAYOUT_I_MAJOR> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;
+
+#if defined(AMD_WMMA_AVAILABLE)
+ static constexpr int ne = tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::ne;
+ nv_bfloat162 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::supported();
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_i(l);
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_j(l);
+ }
+#elif defined(AMD_MFMA_AVAILABLE)
+ static constexpr int ne = tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::ne;
+ nv_bfloat162 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::supported();
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_i(l);
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_j(l);
+ }
+#else
+ static constexpr int ne = I * J / WARP_SIZE;
+ nv_bfloat162 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ if (I == 8 && J == 8) return true;
+ if (I == 16 && J == 4) return true;
+ if (I == 16 && J == 8) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 8 && J == 8) {
+ return threadIdx.x / 4;
+ } else if constexpr (I == 16 && J == 4) {
+ return (l * 8) + (threadIdx.x / 4);
+ } else if constexpr (I == 16 && J == 8) {
+ return ((l % 2) * 8) + (threadIdx.x / 4);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 8 && J == 8) {
+ return (l * 4) + (threadIdx.x % 4);
+ } else if constexpr (I == 16 && J == 4) {
+ return threadIdx.x % 4;
+ } else if constexpr (I == 16 && J == 8) {
+ return ((l / 2) * 4) + (threadIdx.x % 4);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#endif // defined(AMD_WMMA_AVAILABLE)
+ };
+
+ template <int I_, int J_, typename T>
+ struct tile<I_, J_, T, DATA_LAYOUT_J_MAJOR> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_J_MAJOR;
+
+ static constexpr int ne = tile<I_, J_, T, DATA_LAYOUT_I_MAJOR>::ne;
+ T x[ne] = {0};
+
+ static constexpr __device__ bool supported() {
+ return tile<I_, J_, T, DATA_LAYOUT_I_MAJOR>::supported();
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ return tile<I_, J_, T, DATA_LAYOUT_I_MAJOR>::get_j(l);
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ return tile<I_, J_, T, DATA_LAYOUT_I_MAJOR>::get_i(l);
+ }
+ };
+
+ template <int I_, int J_, typename T>
+ struct tile<I_, J_, T, DATA_LAYOUT_I_MAJOR_MIRRORED> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;
+
+ // RDNA3
+ static constexpr int ne = I * J / 32 * 2;
+
+ T x[ne] = {0};
+
+ static constexpr __device__ bool supported() {
+ if (I == 16 && J == 16) return true;
+ if (I == 16 && J == 8) return true;
+ if (I == 16 && J == 4) return true;
+ if (I == 32 && J == 8) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 16) {
+ return threadIdx.x % 16;
+ } else if constexpr (I == 32) {
+ return (threadIdx.x % 16) * 2 + l / (ne/2);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 16) {
+ return l;
+ } else if constexpr (I == 32) {
+ return l % (ne/2);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+ };
+
+ template <int I_, int J_>
+ struct tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;
+#if defined(RDNA3)
+ static constexpr int ne = tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::ne;
+
+ half2 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::supported();
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::get_i(l);
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::get_j(l);
+ }
+#else // Volta
+ static constexpr int ne = I * J / (WARP_SIZE/4);
+
+ half2 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ if (I == 8 && J == 4) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int /*l*/) {
+ if constexpr (I == 8 && J == 4) {
+ return ((threadIdx.x / 16) * 4) + (threadIdx.x % 4);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 8 && J == 4) {
+ return l;
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+#endif // defined(RDNA3)
+ };
+
+ template <int I_, int J_>
+ struct tile<I_, J_, nv_bfloat162, DATA_LAYOUT_I_MAJOR_MIRRORED> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;
+ static constexpr int ne = tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::ne;
+
+ nv_bfloat162 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::supported();
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::get_i(l);
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::get_j(l);
+ }
+ };
+
+ template <int I_, int J_>
+ struct tile<I_, J_, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_J_MAJOR_MIRRORED;
+ static constexpr int ne = I * J / (WARP_SIZE/4);
+
+ half2 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ if (I == 8 && J == 4) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ if constexpr (I == 8 && J == 4) {
+ return ((l / 2) * 4) + (threadIdx.x % 4);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+
+ static __device__ __forceinline__ int get_j(const int l) {
+ if constexpr (I == 8 && J == 4) {
+ return ((threadIdx.x / 16) * 2) + (l % 2);
+ } else {
+ NO_DEVICE_CODE;
+ return -1;
+ }
+ }
+ };
+
+ template <int I_, int J_>
+ struct tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> {
+ static constexpr int I = I_;
+ static constexpr int J = J_;
+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_SCRAMBLED;
+
+ static constexpr int ne = I * J / ggml_cuda_get_physical_warp_size();
+ half2 x[ne] = {{0.0f, 0.0f}};
+
+ static constexpr __device__ bool supported() {
+ if (I == 16 && J == 16) return true;
+ return false;
+ }
+
+ static __device__ __forceinline__ int get_i(const int l) {
+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_i(l);
+ }
+ };
+
+ static __device__ __forceinline__ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> unscramble(const tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & t) {
+#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
+ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> ret;
+#pragma unroll
+ for (int l0 = 0; l0 < t.ne/2; ++l0) {
+ ret.x[2*l0 + 0] = __lows2half2(t.x[l0], t.x[l0 + t.ne/2]);
+ ret.x[2*l0 + 1] = __highs2half2(t.x[l0], t.x[l0 + t.ne/2]);
+ }
+ return ret;
+#else
+ NO_DEVICE_CODE;
+ GGML_UNUSED(t);
+#endif // defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
+ }
+
+#if defined(TURING_MMA_AVAILABLE)
+ template <int I, int J>
+ static __device__ __forceinline__ tile<I, J/2, half2> get_half2(const tile<I, J, float> & tile_float) {
+ tile<I, J/2, half2> ret;
+#pragma unroll
+ for (int l0 = 0; l0 < tile_float.ne; l0 += 2) {
+ ret.x[l0/2] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);
+ }
+ return ret;
+ }
+
+ static __device__ __forceinline__ tile<8, 8, half2> get_transposed(const tile<16, 4, half2> & t) {
+ tile<8, 8, half2> ret;
+ ret.x[0] = ggml_cuda_movmatrix(t.x[0]);
+ ret.x[1] = ggml_cuda_movmatrix(t.x[1]);
+
+ return ret;
+ }
+#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
+ static __device__ __forceinline__ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> get_half2(
+ const tile<16, 16, float, DATA_LAYOUT_I_MAJOR> & tile_float) {
+ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> ret;
+#pragma unroll
+ for (int l = 0; l < tile_float.ne; ++l) {
+ float tmp[2];
+ int i = threadIdx.x / 16;
+ tmp[i] = tile_float.x[l];
+ i ^= 1;
+ tmp[i] = __shfl_xor_sync(0xFFFFFFFF, tile_float.x[l], 16, WARP_SIZE);
+ ret.x[l] = make_half2(tmp[0], tmp[1]);
+ }
+ return ret;
+ }
+#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
+ template <int I, int J>
+ static __device__ __forceinline__ tile<I, J/2, half2> get_half2(const tile<I, J, float> & tile_float) {
+ tile<I, J/2, half2> ret;
+#pragma unroll
+ for (int l0 = 0; l0 < tile_float.ne; l0 += 2) {
+ ret.x[l0/2] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);
+ }
+ return ret;
+ }
+
+ static __device__ __forceinline__ tile<8, 8, half2> get_transposed(const tile<16, 4, half2> & t) {
+ NO_DEVICE_CODE;
+ return tile<8, 8, half2>{};
+ }
+#else // Volta
+ template <int I, int J>
+ static __device__ __forceinline__ tile<I, J/2, half2> get_half2(const tile<I, J, float> & tile_float) {
+ tile<I, J/2, half2> ret;
+#pragma unroll
+ for (int l0 = 0; l0 < tile_float.ne; l0 += 4) {
+ ret.x[l0/2 + 0] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);
+ ret.x[l0/2 + 1] = make_half2(tile_float.x[l0 + 2], tile_float.x[l0 + 3]);
+
+ // On Volta FP16 and FP32 tiles have a different memory layout,
+ // for the conversion threads with an offset of 2 need to exchange half their values:
+ ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)] = __shfl_xor_sync(
+ 0xFFFFFFFF, ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)], 2, WARP_SIZE);
+ }
+ return ret;
+ }
+#endif // defined(TURING_MMA_AVAILABLE)
+
+ template <int I, int J, typename T, data_layout dl>
+ static __device__ __forceinline__ void load_generic(tile<I, J, T, dl> & t, const T * __restrict__ xs0, const int stride) {
+#pragma unroll
+ for (int l = 0; l < t.ne; ++l) {
+ t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)];
+ }
+ }
+
+ template <int stride, typename T>
+ static __device__ __forceinline__ uint32_t swizzle(const uint32_t offset, const uint32_t i) {
+ static_assert(sizeof(T) <= 4, "unsupported type size");
+ constexpr int stride_bytes = stride*sizeof(T);
+ static_assert(stride_bytes % 16 == 0, "bad stride");
+ constexpr uint32_t shift = sizeof(T) == 1 ? 4 : (sizeof(T) == 2 ? 3 : 2);
+ if (stride_bytes % 32 != 0) {
+ return offset; // Equivalent to padding with 16 bytes.
+ }
+ if (stride_bytes % 64 != 0) {
+ return offset ^ (((i / 4) % 2) << shift);
+ }
+ if (stride_bytes % 128 != 0) {
+ return offset ^ (((i / 2) % 4) << shift);
+ }
+ return offset ^ ((i % 8) << shift);
+ }
+
+ template <int stride, typename T>
+ static __device__ __forceinline__ T * swizzle(T * ptr, const uint32_t offset, const uint32_t i) {
+ return ptr + swizzle<stride, T>(offset, i);
+ }
+
+ template <typename T>
+ static __device__ __forceinline__ void load_ldmatrix(
+ tile<8, 8, T> & t, const T * __restrict__ xs0, const int stride) {
+#ifdef TURING_MMA_AVAILABLE
+ int * xi = (int *) t.x;
+ const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + ((threadIdx.x / t.I) * (t.J / 2)) % t.J;
+ asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];"
+ : "=r"(xi[0]), "=r"(xi[1])
+ : "l"(xs));
+#else
+ GGML_UNUSED_VARS(t, xs0, stride);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ template <typename T, data_layout dl>
+ static __device__ __forceinline__ void load_ldmatrix(
+ tile<16, 4, T, dl> & t, const T * __restrict__ xs0, const int stride) {
+#ifdef TURING_MMA_AVAILABLE
+ int * xi = (int *) t.x;
+ const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride;
+ asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];"
+ : "=r"(xi[0]), "=r"(xi[1])
+ : "l"(xs));
+#elif defined(AMD_WMMA_AVAILABLE)
+#ifdef RDNA3
+ static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");
+ static_assert(sizeof(t.x) == 16, "bad ne");
+ ggml_cuda_memcpy_1<8>(t.x + 0, xs0 + t.get_i(0)*stride + 0);
+ ggml_cuda_memcpy_1<8>(t.x + 2, xs0 + t.get_i(0)*stride + 2);
+#else
+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");
+ static_assert(sizeof(t.x) == 8, "bad ne");
+ ggml_cuda_memcpy_1<8>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0));
+#endif // RDNA3
+#elif defined(AMD_MFMA_AVAILABLE)
+ static_assert(sizeof(t.x) == 4, "bad ne");
+ ggml_cuda_memcpy_1<4>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0));
+#else
+ GGML_UNUSED_VARS(t, xs0, stride);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ template <typename T, data_layout dl>
+ static __device__ __forceinline__ void load_ldmatrix(
+ tile<16, 8, T, dl> & t, const T * __restrict__ xs0, const int stride) {
+#if defined(TURING_MMA_AVAILABLE)
+ int * xi = (int * ) t.x;
+ const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2);
+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];"
+ : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3])
+ : "l"(xs));
+#elif defined(VOLTA_MMA_AVAILABLE)
+ ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 0, xs0 + t.get_i(0)*stride + 0);
+ ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 4, xs0 + t.get_i(4)*stride + 4);
+#elif defined(AMD_WMMA_AVAILABLE)
+#ifdef RDNA3
+ static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");
+ static_assert(sizeof(t.x) == 32, "bad ne");
+ ggml_cuda_memcpy_1<16>(t.x + 0, xs0 + t.get_i(0)*stride + 0);
+ ggml_cuda_memcpy_1<16>(t.x + 4, xs0 + t.get_i(0)*stride + 4);
+#else
+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");
+ static_assert(sizeof(t.x) == 16, "bad ne");
+ ggml_cuda_memcpy_1<16>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0));
+#endif // RDNA3
+#elif defined(AMD_MFMA_AVAILABLE)
+ static_assert(sizeof(t.x) == 8, "bad ne");
+ ggml_cuda_memcpy_1<8>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0));
+#else
+ GGML_UNUSED_VARS(t, xs0, stride);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void load_ldmatrix(
+ tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) {
+ ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride);
+ }
+
+ static __device__ __forceinline__ void load_ldmatrix(
+ tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) {
+#ifdef VOLTA_MMA_AVAILABLE
+#pragma unroll
+ for (int l0 = 0; l0 < t.ne; l0 += 2) {
+ ggml_cuda_memcpy_1<2*sizeof(half2)>(t.x + l0, xs0 + t.get_i(l0)*stride + t.get_j(l0));
+ }
+#else
+ GGML_UNUSED_VARS(t, xs0, stride);
+ NO_DEVICE_CODE;
+#endif // VOLTA_MMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void load_ldmatrix(
+ tile<32, 4, half2> & t, const half2 * __restrict__ xs0, const int stride) {
+#if defined(VOLTA_MMA_AVAILABLE)
+ ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride);
+#else
+ GGML_UNUSED_VARS(t, xs0, stride);
+ NO_DEVICE_CODE;
+#endif // defined(VOLTA_MMA_AVAILABLE)
+ }
+
+ template <int I, typename T, data_layout dl>
+ static __device__ __forceinline__ void load_ldmatrix_trans(
+ tile<I, 8, T, dl> & t, const T * __restrict__ xs0, const int stride) {
+#ifdef TURING_MMA_AVAILABLE
+ static_assert(I == 16, "bad tile width");
+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");
+ int * xi = (int *) t.x;
+ const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2);
+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];"
+ : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3])
+ : "l"(xs));
+#elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");
+ if constexpr (I == 32) {
+#pragma unroll
+ for (int l0 = 0; l0 < t.ne/2; ++l0) {
+ const half2 tmp0 = xs0[(2*t.get_j(l0) + 0)*stride + t.get_i(l0)/2];
+ const half2 tmp1 = xs0[(2*t.get_j(l0) + 1)*stride + t.get_i(l0)/2];
+
+ t.x[l0] = __lows2half2(tmp0, tmp1);
+ t.x[l0 + t.ne/2] = __highs2half2(tmp0, tmp1);
+ }
+ } else {
+ half * xh = (half *) t.x;
+#pragma unroll
+ for (int l = 0; l < t.ne; ++l) {
+ xh[2*l + 0] = ((const half *) xs0)[(2*t.get_j(l) + 0)*(2*stride) + t.get_i(l)];
+ xh[2*l + 1] = ((const half *) xs0)[(2*t.get_j(l) + 1)*(2*stride) + t.get_i(l)];
+ }
+ }
+#else
+ GGML_UNUSED_VARS(t, xs0, stride);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ template <int stride, int I, int J, typename T, data_layout dl>
+ static __device__ __forceinline__ void load_ldmatrix_swizzled(
+ tile<I, J, T, dl> & t, const T * __restrict__ xs0, const int offset) {
+#if defined(TURING_MMA_AVAILABLE)
+ static_assert(I == 16, "bad tile width");
+ static_assert(J == 8, "bad tile height");
+ const int i = threadIdx.x % t.I;
+ const int j = (threadIdx.x / t.I) * (t.J / 2);
+ int offset_ij = offset + i * stride + j;
+ offset_ij = swizzle<stride, T>(offset_ij, i);
+ int * xi = (int *) t.x;
+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];"
+ : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3])
+ : "l"(xs0 + offset_ij));
+#elif defined(VOLTA_MMA_AVAILABLE)
+#pragma unroll
+ for (int o = 0; o < t.ne; o += 4) {
+ const int offset_ij = offset + t.get_i(o) * stride + o;
+ ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + o, swizzle<stride>(xs0, offset_ij, t.get_i(o)));
+ }
+#elif defined(AMD_WMMA_AVAILABLE)
+#ifdef RDNA3
+ static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");
+ static_assert(sizeof(t.x) == 32, "bad ne");
+ static_assert(I == 16, "bad tile width");
+ static_assert(J == 8, "bad tile height");
+#pragma unroll
+ for (int o = 0; o < 8; o += 4) {
+ const int offset_ij = offset + t.get_i(0) * stride + o;
+ ggml_cuda_memcpy_1<16>(t.x + o, swizzle<stride>(xs0, offset_ij, t.get_i(0)));
+ }
+#else
+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");
+ static_assert(sizeof(t.x) == 16, "bad ne");
+ const int offset_ij = offset + t.get_i(0)*stride + t.get_j(0);
+ ggml_cuda_memcpy_1<16>(t.x, swizzle<stride>(xs0, offset_ij, t.get_i(0)));
+#endif // RDNA3
+#elif defined(AMD_MFMA_AVAILABLE)
+ static_assert(sizeof(t.x) == 8, "bad ne");
+ const int offset_ij = offset + t.get_i(0)*stride + t.get_j(0);
+ ggml_cuda_memcpy_1<8>(t.x, swizzle<stride>(xs0, offset_ij, t.get_i(0)));
+#else
+ GGML_UNUSED_VARS(t, xs0, offset);
+ NO_DEVICE_CODE;
+#endif // defined(TURING_MMA_AVAILABLE)
+ }
+
+ template <int stride>
+ static __device__ __forceinline__ void load_ldmatrix_swizzled(
+ tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int offset) {
+#ifdef VOLTA_MMA_AVAILABLE
+#pragma unroll
+ for (int l0 = 0; l0 < t.ne; l0 += 2) {
+ const int offset_ij = offset + t.get_i(l0)*stride + t.get_j(l0);
+ ggml_cuda_memcpy_1<2*sizeof(half2)>(t.x + l0, swizzle<stride>(xs0, offset_ij, t.get_i(l0)));
+ }
+#else
+ GGML_UNUSED_VARS(t, xs0, offset);
+ NO_DEVICE_CODE;
+#endif // VOLTA_MMA_AVAILABLE
+ }
+
+ template <int stride, int I, typename T, data_layout dl>
+ static __device__ __forceinline__ void load_ldmatrix_trans_swizzled(
+ tile<I, 8, T, dl> & t, const T * __restrict__ xs0, const int offset) {
+#if defined(TURING_MMA_AVAILABLE)
+ static_assert(I == 16, "bad tile width");
+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");
+ const int i = threadIdx.x % t.I;
+ const int j = (threadIdx.x / t.I) * (t.J / 2);
+ int offset_ij = offset + i * stride + j;
+ offset_ij = swizzle<stride, T>(offset_ij, i);
+ int * xi = (int *) t.x;
+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];"
+ : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3])
+ : "l"(xs0 + offset_ij));
+#elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");
+ if constexpr (I == 32) {
+#pragma unroll
+ for (int l0 = 0; l0 < t.ne/2; ++l0) {
+ half2 tmp[2];
+#pragma unroll
+ for (int o = 0; o < 2; ++o) {
+ const int j = 2*t.get_j(l0) + o;
+ int offset_ij = offset + j*stride + t.get_i(l0)/2;
+ offset_ij = swizzle<stride, T>(offset_ij, j);
+ tmp[o] = xs0[offset_ij];
+ }
+
+ t.x[l0] = __lows2half2(tmp[0], tmp[1]);
+ t.x[l0 + t.ne/2] = __highs2half2(tmp[0], tmp[1]);
+ }
+ } else {
+ half * xh = (half *) t.x;
+#pragma unroll
+ for (int l = 0; l < t.ne; ++l) {
+#pragma unroll
+ for (int o = 0; o < 2; ++o) {
+ const int j = 2*t.get_j(l) + o;
+ xh[2*l + o] = ((const half *) xs0)[swizzle<2*stride, half>(2*offset + j*(2*stride) + t.get_i(l), j)];
+ }
+ }
+ }
+#else
+ GGML_UNUSED_VARS(t, xs0, offset);
+ NO_DEVICE_CODE;
+#endif // defined(TURING_MMA_AVAILABLE)
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<16, 8, int> & D, const tile<16, 4, int> & A, const tile<8, 4, int> & B) {
+#ifdef TURING_MMA_AVAILABLE
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+ asm("mma.sync.aligned.m16n8k16.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"
+ : "+r"(D.x[0]), "+r"(D.x[1]), "+r"(D.x[2]), "+r"(D.x[3])
+ : "r"(A.x[0]), "r"(A.x[1]), "r"(B.x[0]));
+#else
+ // On Turing m16n8k16 mma is not available, use 2x m8n8k16 mma instead:
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(D.x[0]), "+r"(D.x[1])
+ : "r"(A.x[0]), "r"(B.x[0]));
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(D.x[2]), "+r"(D.x[3])
+ : "r"(A.x[1]), "r"(B.x[0]));
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<16, 8, int> & D, const tile<16, 8, int> & A, const tile<8, 8, int> & B) {
+#ifdef TURING_MMA_AVAILABLE
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+ asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"
+ : "+r"(D.x[0]), "+r"(D.x[1]), "+r"(D.x[2]), "+r"(D.x[3])
+ : "r"(A.x[0]), "r"(A.x[1]), "r"(A.x[2]), "r"(A.x[3]), "r"(B.x[0]), "r"(B.x[1]));
+#else
+ // On Turing m16n8k32 mma is not available, use 4x m8n8k16 mma instead:
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(D.x[0]), "+r"(D.x[1])
+ : "r"(A.x[0]), "r"(B.x[0]));
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(D.x[2]), "+r"(D.x[3])
+ : "r"(A.x[1]), "r"(B.x[0]));
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(D.x[0]), "+r"(D.x[1])
+ : "r"(A.x[2]), "r"(B.x[1]));
+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"
+ : "+r"(D.x[2]), "+r"(D.x[3])
+ : "r"(A.x[3]), "r"(B.x[1]));
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<16, 4, half2> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) {
+#ifdef TURING_MMA_AVAILABLE
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ int * Dxi = (int *) D.x;
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+ asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]));
+#else
+ // On Turing m16n8k16 mma is not available, use 2x m8n8k8 mma instead:
+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]));
+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1])
+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]));
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<16, 8, half2> & D, const tile<16, 8, half2> & A, const tile<16, 8, half2> & B) {
+#ifdef TURING_MMA_AVAILABLE
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ int * Dxi = (int *) D.x;
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+ asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[2]));
+ asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};"
+ : "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]), "r"(Bxi[3]));
+#else
+ // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead:
+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]));
+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1])
+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]));
+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"
+ : "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[1]));
+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"
+ : "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3]));
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#elif defined(AMD_WMMA_AVAILABLE)
+#if defined(RDNA4)
+ using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;
+ halfx8_t& acc_frag = reinterpret_cast<halfx8_t&>(D.x[0]);
+ const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]);
+ const halfx8_t& b_frag = reinterpret_cast<const halfx8_t&>(B.x[0]);
+ acc_frag = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag);
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // defined(RDNA4)
+#elif defined(AMD_MFMA_AVAILABLE)
+ // MFMA: FP16 input, FP32 accumulate, convert back to half2.
+ using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;
+ using floatx4_t = __attribute__((ext_vector_type(4))) float;
+
+ // Convert existing half2 accumulator to float for MFMA:
+ floatx4_t acc_f32;
+ {
+ const halfx4_t acc_h = reinterpret_cast<const halfx4_t&>(D.x[0]);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ acc_f32[i] = (float)acc_h[i];
+ }
+ }
+
+ const halfx4_t& a_frag = reinterpret_cast<const halfx4_t&>(A.x[0]);
+ const halfx4_t& b_frag = reinterpret_cast<const halfx4_t&>(B.x[0]);
+ acc_f32 = __builtin_amdgcn_mfma_f32_16x16x16f16(a_frag, b_frag, acc_f32, 0, 0, 0);
+
+ // Convert back to half2:
+ {
+ halfx4_t result_h;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ result_h[i] = (_Float16)acc_f32[i];
+ }
+ reinterpret_cast<halfx4_t&>(D.x[0]) = result_h;
+ }
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR> & A,
+ const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR> & B) {
+#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
+ tile<16, 8, half2> * D16 = (tile<16, 8, half2> *) &D;
+ const tile<16, 8, half2> * A16 = (const tile<16, 8, half2> *) &A;
+ mma(D16[0], A16[0], B);
+ mma(D16[1], A16[1], B);
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)
+ }
+
+ template <data_layout dl_ab, data_layout dl_d>
+ static __device__ __forceinline__ void mma(
+ tile<16, 8, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<8, 8, float, dl_ab> & B) {
+#ifdef AMPERE_MMA_AVAILABLE
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ int * Dxi = (int *) D.x;
+ asm("mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]));
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // AMPERE_MMA_AVAILABLE
+ }
+
+ template <data_layout dl_ab, data_layout dl_d>
+ static __device__ __forceinline__ void mma(
+ tile<16, 16, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<16, 8, float, dl_ab> & B) {
+#ifdef AMD_MFMA_AVAILABLE
+ using floatx4_t = __attribute__((ext_vector_type(4))) float;
+ floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]);
+#if defined(CDNA3)
+ using floatx2_t = __attribute__((ext_vector_type(2))) float;
+ const floatx2_t& a_frag = reinterpret_cast<const floatx2_t&>(A.x[0]);
+ const floatx2_t& b_frag = reinterpret_cast<const floatx2_t&>(B.x[0]);
+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x8_xf32(a_frag, b_frag, acc_frag, 0, 0, 0);
+#elif defined(CDNA4) || defined(CDNA2) || defined(CDNA1)
+ // CDNA4 (gfx950) does not support xf32 MFMA, use f32 path like CDNA2/CDNA1
+#pragma unroll
+ for (int i = 0; i < 2; ++i) {
+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x4f32(A.x[i], B.x[i], acc_frag, 0, 0, 0);
+ }
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // defined(CDNA3)
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // AMD_MFMA_AVAILABLE
+ }
+
+ template <ggml_type type>
+ static __device__ __forceinline__ void mma_block_scaled_fp4(tile<16, 8, float> & D,
+ const tile<16, 8, int> & A,
+ const tile<8, 8, int> & B,
+ uint32_t a_scale,
+ uint32_t b_scale) {
+#ifdef BLACKWELL_MMA_AVAILABLE
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ float * Dxi = (float *) D.x;
+
+ if constexpr (type == GGML_TYPE_MXFP4) {
+ asm volatile(
+ "mma.sync.aligned.kind::mxf4.block_scale.scale_vec::2X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue8m0 "
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, "
+ "%10, {0, 0}, %11, {0, 0};"
+ : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale));
+ } else {
+ asm volatile(
+ "mma.sync.aligned.kind::mxf4nvf4.block_scale.scale_vec::4X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue4m3 "
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, "
+ "%10, {0, 0}, %11, {0, 0};"
+ : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale));
+ }
+#else
+ GGML_UNUSED_VARS(D, A, B, a_scale, b_scale);
+#endif // BLACKWELL_MMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<16, 8, float> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) {
+#ifdef TURING_MMA_AVAILABLE
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ int * Dxi = (int *) D.x;
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+ asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]));
+#else
+ // On Turing m16n8k16 mma is not available, use 2x m8n8k8 mma instead:
+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]));
+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]));
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<16, 8, float> & D, const tile<16, 8, nv_bfloat162> & A, const tile<8, 8, nv_bfloat162> & B) {
+#ifdef AMPERE_MMA_AVAILABLE
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ int * Dxi = (int *) D.x;
+ asm("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]));
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // AMPERE_MMA_AVAILABLE
+ }
+
+ template <data_layout dl_ab, data_layout dl_d>
+ static __device__ __forceinline__ void mma(
+ tile<16, 16, float, dl_d> & D, const tile<16, 8, half2, dl_ab> & A, const tile<16, 8, half2, dl_ab> & B) {
+#ifdef TURING_MMA_AVAILABLE
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ int * Dxi = (int *) D.x;
+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+ asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[2]));
+ asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]), "r"(Bxi[3]));
+#else
+ // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead:
+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]));
+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]));
+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[1]));
+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])
+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3]));
+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#elif defined(AMD_WMMA_AVAILABLE)
+#if defined(RDNA4)
+ using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;
+ using floatx8_t = __attribute__((ext_vector_type(8))) float;
+ floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
+ const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]);
+ const halfx8_t& b_frag = reinterpret_cast<const halfx8_t&>(B.x[0]);
+ acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag);
+#elif defined(RDNA3)
+ using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;
+ using floatx8_t = __attribute__((ext_vector_type(8))) float;
+ floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
+ const halfx16_t& a_frag = reinterpret_cast<const halfx16_t&>(A.x[0]);
+ const halfx16_t& b_frag = reinterpret_cast<const halfx16_t&>(B.x[0]);
+ acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a_frag, b_frag, acc_frag);
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // RDNA4
+#elif defined(AMD_MFMA_AVAILABLE)
+ using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;
+ using floatx4_t = __attribute__((ext_vector_type(4))) float;
+ floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]);
+ const halfx4_t& a_frag = reinterpret_cast<const halfx4_t&>(A.x[0]);
+ const halfx4_t& b_frag = reinterpret_cast<const halfx4_t&>(B.x[0]);
+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x16f16(a_frag, b_frag, acc_frag, 0, 0, 0);
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ template <data_layout dl_ab, data_layout dl_d>
+ static __device__ __forceinline__ void mma(
+ tile<16, 16, float, dl_d> & D, const tile<16, 8, nv_bfloat162, dl_ab> & A, const tile<16, 8, nv_bfloat162, dl_ab> & B) {
+#if defined(AMD_WMMA_AVAILABLE)
+#if defined(RDNA4)
+ using bf16x8_t = __attribute__((ext_vector_type(8))) __bf16;
+ using floatx8_t = __attribute__((ext_vector_type(8))) float;
+ floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
+ const bf16x8_t& a_frag = reinterpret_cast<const bf16x8_t&>(A.x[0]);
+ const bf16x8_t& b_frag = reinterpret_cast<const bf16x8_t&>(B.x[0]);
+ acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12(a_frag, b_frag, acc_frag);
+#elif defined(RDNA3)
+ using bf16x16_t = __attribute__((ext_vector_type(16))) __bf16;
+ using floatx8_t = __attribute__((ext_vector_type(8))) float;
+ floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
+ const bf16x16_t& a_frag = reinterpret_cast<const bf16x16_t&>(A.x[0]);
+ const bf16x16_t& b_frag = reinterpret_cast<const bf16x16_t&>(B.x[0]);
+ acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32(a_frag, b_frag, acc_frag);
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // defined(RDNA4)
+#elif defined(AMD_MFMA_AVAILABLE)
+ using floatx4_t = __attribute__((ext_vector_type(4))) float;
+ floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]);
+#if defined(CDNA4) || defined(CDNA3) || defined(CDNA2)
+ using bf16x4_t = __attribute__((ext_vector_type(4))) __bf16;
+ const bf16x4_t& a_frag = reinterpret_cast<const bf16x4_t&>(A.x[0]);
+ const bf16x4_t& b_frag = reinterpret_cast<const bf16x4_t&>(B.x[0]);
+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x16bf16_1k(a_frag, b_frag, acc_frag, 0, 0, 0);
+#elif defined(CDNA1)
+#pragma unroll
+ for (int i = 0; i < 2; ++i) {
+ using bf16x2_t = __attribute__((ext_vector_type(2))) __bf16;
+ const bf16x2_t& a_frag = reinterpret_cast<const bf16x2_t&>(A.x[i]);
+ const bf16x2_t& b_frag = reinterpret_cast<const bf16x2_t&>(B.x[i]);
+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x8bf16(a_frag, b_frag, acc_frag, 0, 0, 0);
+ }
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // defined(CDNA3) || defined(CDNA2)
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // defined(AMD_WMMA_AVAILABLE)
+ }
+
+ template <data_layout dl_d, data_layout dl_ab>
+ static __device__ __forceinline__ void mma(
+ tile<16, 16, int, dl_d> & D, const tile<16, 8, int, dl_ab> & A, const tile<16, 8, int, dl_ab> & B) {
+#if defined(AMD_MFMA_AVAILABLE)
+ using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;
+ int32x4_t * acc = (int32x4_t *) D.x;
+#if defined(CDNA4) || defined(CDNA3)
+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(((int64_t *) A.x)[0], ((int64_t *) B.x)[0], acc[0], 0, 0, 0);
+#elif defined(CDNA2) || defined(CDNA1)
+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], B.x[0], acc[0], 0, 0, 0);
+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[1], B.x[1], acc[0], 0, 0, 0);
+#endif // defined(CDNA4) || defined(CDNA3)
+#elif defined(AMD_WMMA_AVAILABLE)
+ using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;
+ int32x8_t * acc = (int32x8_t *) D.x;
+#if defined(RDNA4)
+ using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;
+ int32x2_t * a_vec = (int32x2_t *) A.x;
+ int32x2_t * b_vec = (int32x2_t *) B.x;
+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], true);
+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[1], true, b_vec[1], acc[0], true);
+#elif defined(RDNA3)
+ using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;
+ int32x4_t * a_vec = (int32x4_t *) A.x;
+ int32x4_t * b_vec = (int32x4_t *) B.x;
+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], true);
+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[1], true, b_vec[1], acc[0], true);
+#endif // RDNA4
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // AMD_MFMA_AVAILABLE
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<32, 32, int> & D, const tile<32, 4, int> & A, const tile<32, 4, int> & B) {
+#if defined(AMD_MFMA_AVAILABLE)
+ using int32x16_t = __attribute__((__vector_size__(16 * sizeof(int)))) int;
+ int32x16_t * acc = (int32x16_t *) D.x;
+#if defined(CDNA4) || defined(CDNA3)
+ acc[0] = __builtin_amdgcn_mfma_i32_32x32x16_i8(((int64_t *) A.x)[0], ((int64_t *) B.x)[0], acc[0], 0, 0, 0);
+#elif defined(CDNA2) || defined(CDNA1)
+ acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[0], B.x[0], acc[0], 0, 0, 0);
+ acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[1], B.x[1], acc[0], 0, 0, 0);
+#endif // defined(CDNA4) || defined(CDNA3)
+
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // AMD_MFMA_AVAILABLE
+ }
+
+ template <typename T1, typename T2, int J, int K>
+ static __device__ __forceinline__ void mma(
+ tile<32, J, T1> & D, const tile<32, K, T2> & A, const tile<J, K, T2> & B) {
+ tile <16, J, T1> * D16 = reinterpret_cast< tile<16, J, T1> *>(&D);
+ const tile<16, K, T2> * A16 = reinterpret_cast<const tile<16, K, T2> *>(&A);
+ mma(D16[0], A16[0], B);
+ mma(D16[1], A16[1], B);
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<32, 8, float> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) {
+#if defined(VOLTA_MMA_AVAILABLE)
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ int * Dxi = (int *) D.x;
+ asm("mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 "
+ "{%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9}, {%10, %11}, {%0, %1, %2, %3, %4, %5, %6, %7};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]), "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]), "r"(Bxi[1]));
+ asm("mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 "
+ "{%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9}, {%10, %11}, {%0, %1, %2, %3, %4, %5, %6, %7};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]), "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])
+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]), "r"(Bxi[3]));
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // defined(VOLTA_MMA_AVAILABLE)
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<32, 4, half2> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & B) {
+#if defined(VOLTA_MMA_AVAILABLE)
+ const int * Axi = (const int *) A.x;
+ const int * Bxi = (const int *) B.x;
+ int * Dxi = (int *) D.x;
+ asm("mma.sync.aligned.m8n8k4.row.row.f16.f16.f16.f16 "
+ "{%0, %1, %2, %3}, {%4, %5}, {%6, %7}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]), "r"(Bxi[1]));
+ asm("mma.sync.aligned.m8n8k4.row.row.f16.f16.f16.f16 "
+ "{%0, %1, %2, %3}, {%4, %5}, {%6, %7}, {%0, %1, %2, %3};"
+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])
+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]), "r"(Bxi[3]));
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // defined(VOLTA_MMA_AVAILABLE)
+ }
+
+ static __device__ __forceinline__ void mma(
+ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & A,
+ const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) {
+#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
+ using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;
+ halfx16_t * xD = (halfx16_t *) D.x;
+ const halfx16_t * xA = (const halfx16_t *) A.x;
+ const halfx16_t * xB = (const halfx16_t *) B.x;
+ xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[0], xB[0], xD[0], /*opsel =*/ 0);
+ xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[1], xB[0], xD[0], /*opsel =*/ 1);
+#else
+ GGML_UNUSED_VARS(D, A, B);
+ NO_DEVICE_CODE;
+#endif // TURING_MMA_AVAILABLE
+ }
+
+ template <data_layout dl_d, data_layout dl_ab>
+ static __device__ __forceinline__ void mma(
+ tile<16, 16, int, dl_d> & D, const tile<16, 4, int, dl_ab> & A, const tile<16, 4, int, dl_ab> & B) {
+#if defined(AMD_MFMA_AVAILABLE)
+ using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;
+ int32x4_t * acc = (int32x4_t *) D.x;
+#if defined(CDNA4) || defined(CDNA3)
+ const int64_t xA = uint32_t(A.x[0]);
+ const int64_t xB = uint32_t(B.x[0]);
+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(xA, xB, acc[0], 0, 0, 0);
+#elif defined(CDNA2) || defined(CDNA1)
+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], B.x[0], acc[0], 0, 0, 0);
+#endif // defined(CDNA4) || defined(CDNA3)
+#elif defined(AMD_WMMA_AVAILABLE)
+ using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;
+ int32x8_t * acc = (int32x8_t *) D.x;
+#if defined(RDNA4)
+ using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;
+ int32x2_t * a_vec = (int32x2_t *) A.x;
+ int32x2_t * b_vec = (int32x2_t *) B.x;
+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], false);
+#elif defined(RDNA3)
+ using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;
+ int32x4_t * a_vec = (int32x4_t *) A.x;
+ int32x4_t * b_vec = (int32x4_t *) B.x;
+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], false);
+#endif // RDNA4
+#else
+ GGML_UNUSED(D);
+ GGML_UNUSED(A);
+ GGML_UNUSED(B);
+ NO_DEVICE_CODE;
+#endif // AMD_WMMA_AVAILABLE
+ }
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cu
new file mode 100644
index 00000000..345b39eb
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cu
@@ -0,0 +1,221 @@
+#include "common.cuh"
+#include "mmid.cuh"
+
+// To reduce shared memory use, store "it" and "iex_used" with 22/10 bits each.
+struct mm_ids_helper_store {
+ uint32_t data;
+
+ __device__ mm_ids_helper_store(const uint32_t it, const uint32_t iex_used) {
+ data = (it & 0x003FFFFF) | (iex_used << 22);
+ }
+
+ __device__ uint32_t it() const {
+ return data & 0x003FFFFF;
+ }
+
+ __device__ uint32_t iex_used() const {
+ return data >> 22;
+ }
+};
+static_assert(sizeof(mm_ids_helper_store) == 4, "unexpected size for mm_ids_helper_store");
+
+// the generic path passes 0, which needs no padding since it never groups lanes by token
+template <int n> struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2<(n + 1)/2>::value; };
+template <> struct mm_ids_pow2<1> { static constexpr int value = 1; };
+template <> struct mm_ids_pow2<0> { static constexpr int value = 1; };
+
+// Helper function for mul_mat_id, converts ids to a more convenient format.
+// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert.
+// ids_dst describes the same mapping but for the dst tensor.
+// The upper and lower bounds for the ith expert in the compact src1 tensor are stored in expert_bounds[i:i+1].
+template <int n_expert_used_template>
+__launch_bounds__(ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mm_ids_helper(
+ const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,
+ const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template;
+ const int expert = blockIdx.x;
+
+ extern __shared__ char data_mm_ids_helper[];
+ mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper;
+
+ int nex_prev = 0; // Number of columns for experts with a lower index.
+ int it_compact = 0; // Running index for the compact slice of this expert.
+
+ if constexpr (n_expert_used_template == 0) {
+ // Generic implementation:
+ for (int it = 0; it < n_tokens; ++it) {
+ int iex_used = -1; // The index at which the expert is used, if any.
+ for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) {
+ const int expert_used = ids[it*si1 + iex];
+ nex_prev += expert_used < expert;
+ if (expert_used == expert) {
+ iex_used = iex;
+ }
+ }
+
+ if (iex_used != -1) {
+ store[it_compact] = mm_ids_helper_store(it, iex_used);
+ }
+
+ if (warp_reduce_any<warp_size>(iex_used != -1)) {
+ it_compact++;
+ }
+ }
+ } else {
+ // Implementation optimized for specific numbers of experts used:
+ // a warp holds a whole number of token slots, so the slot count is padded to a power of 2
+ constexpr int neu_padded = mm_ids_pow2<n_expert_used_template>::value;
+ static_assert(neu_padded <= warp_size && warp_size % neu_padded == 0, "bad n_expert_used");
+ for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) {
+ const int it = it0 + threadIdx.x / neu_padded;
+
+ const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
+ const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
+ ids[it*si1 + iex] : INT_MAX;
+ const int iex_used = expert_used == expert ? iex : -1;
+ nex_prev += expert_used < expert;
+
+ // Whether the threads at this token position have used the expert:
+ const int it_compact_add_self = warp_reduce_any<neu_padded>(iex_used != -1);
+
+ // Do a scan over threads at lower token positions in warp to get the correct index for writing data:
+ int it_compact_add_lower = 0;
+#pragma unroll
+ for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {
+ const int tmp = __shfl_up_sync(0xFFFFFFFF, it_compact_add_self, offset, warp_size);
+ if (threadIdx.x >= static_cast<unsigned int>(offset)) {
+ it_compact_add_lower += tmp;
+ }
+ }
+
+ if (iex_used != -1) {
+ store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used);
+ }
+
+ // The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:
+ it_compact += __shfl_sync(0xFFFFFFFF, it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);
+ }
+ }
+ nex_prev = warp_reduce_sum<warp_size>(nex_prev);
+ ggml_cuda_syncwarp();
+
+ for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
+ const mm_ids_helper_store store_it = store[itc];
+ const int it = store_it.it();
+ const int iex_used = store_it.iex_used();
+ ids_dst[nex_prev + itc] = it*n_expert_used + iex_used;
+ // ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
+ if (write_inverse) {
+ ids_src1[it*n_expert_used + iex_used] = nex_prev + itc;
+ } else {
+ ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y;
+ }
+ }
+
+ if (threadIdx.x != 0) {
+ return;
+ }
+
+ expert_bounds[expert] = nex_prev;
+
+ if (expert < static_cast<int>(gridDim.x) - 1) {
+ return;
+ }
+
+ expert_bounds[gridDim.x] = nex_prev + it_compact;
+}
+
+template <int n_expert_used_template>
+static void launch_mm_ids_helper(
+ const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,
+ const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) {
+ GGML_ASSERT(n_tokens < (1 << 22) && "too few bits in mm_ids_helper_store");
+ GGML_ASSERT(n_expert_used_var < (1 << 10) && "too few bits in mm_ids_helper_store");
+
+ const int id = ggml_cuda_get_device();
+ const int warp_size = ggml_cuda_info().devices[id].warp_size;
+ const size_t smpbo = ggml_cuda_info().devices[id].smpbo;
+ CUDA_SET_SHARED_MEMORY_LIMIT(mm_ids_helper<n_expert_used_template>, smpbo);
+
+ const dim3 num_blocks(n_experts, 1, 1);
+ const dim3 block_size(warp_size, 1, 1);
+ const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store);
+ GGML_ASSERT(nbytes_shared <= smpbo);
+ mm_ids_helper<n_expert_used_template><<<num_blocks, block_size, nbytes_shared, stream>>>
+ (ids, ids_src1, ids_dst, expert_bounds, n_tokens, n_expert_used_var, nchannels_y, si1, sis1, write_inverse);
+}
+
+void ggml_cuda_launch_mm_ids_helper(
+ const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,
+ const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) {
+ switch (n_expert_used) {
+ case 2:
+ launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
+ break;
+ case 4:
+ launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
+ break;
+ case 6:
+ launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
+ break;
+ case 8:
+ launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
+ break;
+ case 10:
+ launch_mm_ids_helper<10>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
+ break;
+ case 16:
+ launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
+ break;
+ case 32:
+ launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
+ break;
+ default:
+ launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
+ break;
+ }
+}
+
+// Lists the (expert, column tile) pairs that hold columns for a flat MoE grid, in expert order.
+// The other ntiles grid columns get tile jt_none of expert 0, which starts past its columns, so mul_mat_q skips them.
+template <int nthreads>
+__launch_bounds__(nthreads, 1)
+static __global__ void mm_ids_tile_map(
+ const int32_t * __restrict__ expert_bounds, int2 * __restrict__ tile_map, const int n_experts, const int J, const int ntiles, const int jt_none) {
+ __shared__ int scan[nthreads];
+
+ int start = 0; // First tile of the current chunk of experts.
+ for (int e0 = 0; e0 < n_experts; e0 += nthreads) {
+ const int e = e0 + threadIdx.x;
+ const int n = e < n_experts ? (expert_bounds[e + 1] - expert_bounds[e] + J - 1) / J : 0;
+
+ scan[threadIdx.x] = n;
+ __syncthreads();
+#pragma unroll
+ for (int offset = 1; offset < nthreads; offset *= 2) {
+ const int add = threadIdx.x >= offset ? scan[threadIdx.x - offset] : 0;
+ __syncthreads();
+ scan[threadIdx.x] += add;
+ __syncthreads();
+ }
+
+ const int first = start + scan[threadIdx.x] - n;
+ for (int jt = 0; jt < n; ++jt) {
+ tile_map[first + jt] = make_int2(e, jt);
+ }
+ start += scan[nthreads - 1];
+ __syncthreads();
+ }
+
+ for (int t = start + threadIdx.x; t < ntiles; t += nthreads) {
+ tile_map[t] = make_int2(0, jt_none);
+ }
+}
+
+void ggml_cuda_launch_mm_ids_tile_map(
+ const int32_t * expert_bounds, int2 * tile_map, const int n_experts, const int J, const int ntiles, const int jt_none, cudaStream_t stream) {
+ constexpr int nthreads = 512;
+ mm_ids_tile_map<nthreads><<<1, nthreads, 0, stream>>>(expert_bounds, tile_map, n_experts, J, ntiles, jt_none);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cuh
new file mode 100644
index 00000000..b0e8edb8
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cuh
@@ -0,0 +1,8 @@
+#pragma once
+
+void ggml_cuda_launch_mm_ids_helper(
+ const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds,
+ int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream);
+
+void ggml_cuda_launch_mm_ids_tile_map(
+ const int32_t * expert_bounds, int2 * tile_map, int n_experts, int J, int ntiles, int jt_none, cudaStream_t stream);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-ampere.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-ampere.cuh
new file mode 100644
index 00000000..9f9fd197
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-ampere.cuh
@@ -0,0 +1,383 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_ampere(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-blackwell.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-blackwell.cuh
new file mode 100644
index 00000000..9fbe32b6
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-blackwell.cuh
@@ -0,0 +1,37 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_blackwell(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);
+
+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-cdna.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-cdna.cuh
new file mode 100644
index 00000000..4a8d89f7
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-cdna.cuh
@@ -0,0 +1,185 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_cdna(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
+
+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-gcn.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-gcn.cuh
new file mode 100644
index 00000000..24af2ef2
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-gcn.cuh
@@ -0,0 +1,281 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-dp4a.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-dp4a.cuh
new file mode 100644
index 00000000..83eb7c14
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-dp4a.cuh
@@ -0,0 +1,273 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_pascal_dp4a(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-older.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-older.cuh
new file mode 100644
index 00000000..2a8dc9e1
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-older.cuh
@@ -0,0 +1,273 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_pascal_older(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna2.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna2.cuh
new file mode 100644
index 00000000..8324d9e1
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna2.cuh
@@ -0,0 +1,273 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna2(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3-5.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3-5.cuh
new file mode 100644
index 00000000..ead3825d
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3-5.cuh
@@ -0,0 +1,290 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3_5(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3.cuh
new file mode 100644
index 00000000..3a3ef7bd
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3.cuh
@@ -0,0 +1,274 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 4, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 4, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna4.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna4.cuh
new file mode 100644
index 00000000..9293d9d5
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna4.cuh
@@ -0,0 +1,290 @@
+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna4(ggml_type type, int J, bool fallback) {
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+
+// ---------------------------------------------------------------------------------------------
+
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
+
+ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh
new file mode 100644
index 00000000..017b93b5
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh
@@ -0,0 +1,1768 @@
+#pragma once
+
+#include "vecdotq.cuh"
+
+#include "mmq.cuh"
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q1_0(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int blocks_per_iter = MMQ_ITER_K / QK1_0;
+ constexpr int threads_per_row = blocks_per_iter * QI1_0;
+ constexpr int nrows = warp_size / threads_per_row;
+ constexpr int scale_entries_per_block = QK1_0 / QK8_1;
+ constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block;
+
+ const int txi = threadIdx.x % threads_per_row;
+ const int kbx = txi / QI1_0;
+ const int kqsx = txi % QI1_0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + kbx;
+ const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 2;
+
+ const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0;
+#pragma unroll
+ for (int j = 0; j < 2; ++j) {
+ const int q = qxi[j];
+
+ // unpack crumbs into nibble indices
+ const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13]
+ const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15]
+ // unpack nibbles into byte values
+ const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0);
+ const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0);
+ const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16);
+ const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16);
+ // unshuffle values
+ const int v0 = __byte_perm(s0, s1, 0x5410);
+ const int v1 = __byte_perm(s0, s1, 0x7632);
+ const int v2 = __byte_perm(s2, s3, 0x5410);
+ const int v3 = __byte_perm(s2, s3, 0x7632);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + dst_offset + j*4+0] = v0;
+ x_qs[i*sram_stride + dst_offset + j*4+1] = v1;
+ x_qs[i*sram_stride + dst_offset + j*4+2] = v2;
+ x_qs[i*sram_stride + dst_offset + j*4+3] = v3;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+0] = v0;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+1] = v1;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+2] = v2;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+3] = v3;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+ }
+
+ const int ksx = threadIdx.x % scale_entries_per_row;
+ const int scale_block = ksx / scale_entries_per_block;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + scale_block;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + ksx] = bxi->d;
+#else
+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int blocks_per_iter = MMQ_ITER_K / QK2_0;
+ constexpr int threads_per_row = blocks_per_iter * QI2_0;
+ constexpr int nrows = warp_size / threads_per_row;
+ constexpr int scale_entries_per_block = QK2_0 / QK8_1;
+ constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block;
+
+ const int txi = threadIdx.x % threads_per_row;
+ const int kbx = txi / QI2_0;
+ const int kqsx = txi % QI2_0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx;
+ const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 4;
+
+ const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0;
+
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int q = qxi[j];
+
+#if defined(GGML_USE_HIP)
+ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);
+ const uint32_t qy_bits = q >> 8;
+ const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);
+ const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);
+ const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);
+#else
+ // unpack even and odd crumbs into byte values
+ const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);
+ const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);
+ // unshuffle values
+ const int qx = __byte_perm(qe, qo, 0x5140);
+ const int qy = __byte_perm(qe, qo, 0x7362);
+#endif // defined(GGML_USE_HIP)
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + dst_offset + j*2+0] = qx;
+ x_qs[i*sram_stride + dst_offset + j*2+1] = qy;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+0] = qx;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+1] = qy;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+ }
+
+ const int ksx = threadIdx.x % scale_entries_per_row;
+ const int scale_block = ksx / scale_entries_per_block;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps) {
+ int i = i0 + threadIdx.y;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + scale_block;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + ksx] = bxi->d;
+#else
+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_0);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+ const int kbx = txi / QI4_0;
+ const int kqsx = txi % QI4_0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx;
+ const int qs0 = get_int_b2(bxi->qs, kqsx);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + 0] = __vsub4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808);
+ x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsub4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808);
+#else
+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)
+ }
+
+ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_0;
+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {
+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + kbxd] = bxi->d;
+#else
+ x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + kbxd] = bxi->d;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_1(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_1);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+ const int kbx = txi / QI4_1;
+ const int kqsx = txi % QI4_1;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx;
+ const int qs0 = get_int_b4(bxi->qs, kqsx);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F;
+ x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F;
+#else
+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_1;
+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {
+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_dm[i*sram_stride + kbxd] = bxi->dm;
+#else
+ x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + kbxd] = bxi->dm;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_0(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_0);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+ const int kbx = txi / QI5_0;
+ const int kqsx = txi % QI5_0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx;
+
+ const int ql = get_int_b2(bxi->qs, kqsx);
+ const int qh = get_int_b2(bxi->qh, 0) >> (4 * kqsx);
+
+ int qs0 = (ql >> 0) & 0x0F0F0F0F;
+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
+ qs0 = __vsub4(qs0, 0x10101010); // subtract 16
+
+ int qs1 = (ql >> 4) & 0x0F0F0F0F;
+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
+ qs1 = __vsub4(qs1, 0x10101010); // subtract 16
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + 0] = qs0;
+ x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_0;
+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {
+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + kbxd] = bxi->d;
+#else
+ x_df[i*(MMQ_TILE_NE_K/QI5_0) + i/QI5_0 + kbxd] = bxi->d;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_1(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_1);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+ const int kbx = txi / QI5_1;
+ const int kqsx = txi % QI5_1;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx;
+
+ const int ql = get_int_b4(bxi->qs, kqsx);
+ const int qh = get_int_b4(bxi->qh, 0) >> (4 * kqsx);
+
+ int qs0 = (ql >> 0) & 0x0F0F0F0F;
+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
+
+ int qs1 = (ql >> 4) & 0x0F0F0F0F;
+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + 0] = qs0;
+ x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_1;
+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {
+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_dm[i*sram_stride + kbxd] = bxi->dm;
+#else
+ x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + kbxd] = bxi->dm;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ // MMQ_ITER_K / (4 * QR8_0) == 64 required. but NV has only 32 threads per warp
+ constexpr int threads_per_row = 32;
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+ const int kbx = txi / QI8_0;
+ const int kqsx = txi % QI8_0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);
+ x_qs[i*sram_stride + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0;
+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {
+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + kbxd] = bxi->d;
+#else
+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+// ---------------------------------------------------------------------------------------------
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR2_K);
+ constexpr int nrows = ggml_cuda_get_physical_warp_size() / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride;
+
+ const int x_ql_0 = get_int_b2(bxi->qs, kqsx);
+
+#pragma unroll
+ for (int l = 0; l < QR2_K; ++l) {
+ const int k = (kqsx/8)*32 + l*8 + kqsx % 8;
+
+ const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + k] = x_qs_k;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const int sc_m = bxi->scales[kqsx];
+#ifdef FAST_FP16_AVAILABLE
+ const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4));
+#else
+ const float2 bxi_dmf = __half22float2(bxi->dm);
+ const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4));
+#endif // FAST_FP16_AVAILABLE
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_dm[i*sram_stride + kqsx] = x_dm_ik;
+#else
+ x_dm[i*(MMQ_TILE_NE_K + 1) + kqsx] = x_dm_ik;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+ int * x_sc = (int *) (x_df + txs.dm);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR3_K);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;
+
+ const int x_ql_0 = get_int_b2(bxi->qs, kqsx);
+ const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2)));
+
+#pragma unroll
+ for (int l = 0; l < QR3_K; ++l) {
+ const int k = (kqsx/8)*32 + l*8 + kqsx % 8;
+
+ const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303;
+ const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404;
+
+ const int x_qs_k = __vsub4(x_ql_k | x_qh_k, 0x04040404);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + k] = x_qs_k;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+ }
+
+ constexpr int rows_per_warp = warp_size / 4;
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {
+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/4;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;
+
+ const int ksc = threadIdx.x % 4;
+
+ const int ksc_low = ksc % (QI3_K/8);
+ const int shift_low = 4 * (ksc / (QI3_K/8));
+ const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;
+
+ const int ksc_high = QI3_K/8;
+ const int shift_high = 2 * ksc;
+ const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;
+
+ const int sc = __vsub4(sc_low | sc_high, 0x20202020);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ const int8_t * sc8 = (const int8_t *) &sc;
+ const float d = bxi->d;
+
+#pragma unroll
+ for (int l = 0; l < int(sizeof(int)); ++l) {
+ x_df[i*sram_stride + sizeof(int)*ksc + l] = d*sc8[l];
+ }
+#else
+ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = sc;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+#if !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE))
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {
+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;
+
+ x_df[i] = bxi->d;
+ }
+#endif // !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)) || defined(AMD_WMMA_AVAILABLE)
+}
+
+static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) {
+ // scale arrangement after the following two lines:
+ // - ksc == 0: sc0, sc1, sc2, sc3
+ // - ksc == 1: sc4, sc5, sc6, sc7
+ // - ksc == 2: m0, m1, m2, m3
+ // - ksc == 3: m4, m5, m6, m7
+ return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits
+ ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+ int * x_sc = (int *) (x_dm + txs.dm);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_K);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
+ const int qs0 = get_int_b4(bxi->qs, txi);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F;
+ x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F;
+#else
+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int rows_per_warp = warp_size / 2;
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ // Need if on AMD instead of % because warp_size == 64
+ // This causes double work and throughput loss (MI300X)
+ // H100 loses about 100 t/s with 'if' condition over '%'
+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;
+ if (i < I) {
+#else
+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I;
+ {
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
+
+ const int * scales = (const int *) bxi->scales;
+ const int ksc = threadIdx.x % 2;
+
+ const int sc32 = unpack_scales_q45_K(scales, ksc + 0);
+ const int m32 = unpack_scales_q45_K(scales, ksc + 2);
+
+ const uint8_t * sc8 = (const uint8_t *) &sc32;
+ const uint8_t * m8 = (const uint8_t *) &m32;
+
+ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);
+
+ #pragma unroll
+ for (int l = 0; l < int(sizeof(int)); ++l) {
+ x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);
+ }
+ }
+ }
+#else
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {
+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
+
+ x_dm[i] = bxi->dm;
+ }
+ constexpr int rows_per_warp = warp_size / 4;
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {
+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8);
+
+ const int * scales = (const int *) bxi->scales;
+
+ const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);
+ const int scales8 = unpack_scales_q45_K(scales, ksc);
+
+ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;
+ }
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I);
+ int * x_qs = (int *) x_tile;
+ half2 * x_dm = (half2 *) (x_qs + txs.qs);
+ int * x_sc = (int *) (x_dm + txs.dm);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_K);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
+ const int ky = QR5_K*txi;
+
+ const int ql = get_int_b4(bxi->qs, txi);
+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;
+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;
+
+ const int qh = get_int_b4(bxi->qh, txi % (QI5_K/4));
+ const int qh0 = ((qh >> (2 * (txi / (QI5_K/4)) + 0)) << 4) & 0x10101010;
+ const int qh1 = ((qh >> (2 * (txi / (QI5_K/4)) + 1)) << 4) & 0x10101010;
+
+ const int kq0 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + 0;
+ const int kq1 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + QI5_K/4;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + kq0] = ql0 | qh0;
+ x_qs[i*sram_stride + kq1] = ql1 | qh1;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = ql0 | qh0;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = ql1 | qh1;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int rows_per_warp = warp_size / 2;
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {
+#if defined(AMD_MFMA_AVAILABLE)
+ // Need if on AMD instead of % because warp_size == 64
+ // This causes double work and throughput loss (MI300X)
+ // H100 loses about 100 t/s with 'if' condition over '%'
+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;
+ if (i < I) {
+#else
+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I;
+ {
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
+
+ const int * scales = (const int *) bxi->scales;
+ const int ksc = threadIdx.x % 2;
+
+ const int sc32 = unpack_scales_q45_K(scales, ksc + 0);
+ const int m32 = unpack_scales_q45_K(scales, ksc + 2);
+
+ const uint8_t * sc8 = (const uint8_t *) &sc32;
+ const uint8_t * m8 = (const uint8_t *) &m32;
+
+ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);
+
+#pragma unroll
+ for (int l = 0; l < int(sizeof(int)); ++l) {
+ x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);
+ }
+ }
+ }
+#else
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {
+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
+
+ x_dm[i] = bxi->dm;
+ }
+
+ constexpr int rows_per_warp = warp_size / 4;
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {
+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
+
+ const int * scales = (const int *) bxi->scales;
+
+ const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);
+ const int scales8 = unpack_scales_q45_K(scales, ksc);
+
+ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;
+ }
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+ int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+ int * x_sc = (int *) (x_df + txs.dm);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR6_K);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;
+
+ const int ql = get_int_b2(bxi->ql, txi);
+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;
+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;
+
+ const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (txi / (QI6_K/2)) + txi % (QI6_K/4));
+ const int qh0 = ((qh >> ((txi & 0x08) >> 2)) << 4) & 0x30303030;
+ const int qh1 = (qh >> ((txi & 0x08) >> 2)) & 0x30303030;
+
+ const int kq0 = 2*txi - txi % (QI6_K/2) + 0;
+ const int kq1 = 2*txi - txi % (QI6_K/2) + QI6_K/2;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + kq0] = __vsub4(ql0 | qh0, 0x20202020);
+ x_qs[i*sram_stride + kq1] = __vsub4(ql1 | qh1, 0x20202020);
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = __vsub4(ql0 | qh0, 0x20202020);
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = __vsub4(ql1 | qh1, 0x20202020);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {
+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride] = bxi->d;
+#else
+ x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K] = bxi->d;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ constexpr int rows_per_warp = warp_size / 4;
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {
+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8));
+#else
+ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8));
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+// ---------------------------------------------------------------------------------------------
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_s(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I);
+ int * x_qs = (int *) x_tile;
+ half2 * x_ds = (half2 *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_S);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride;
+
+ const int qs_packed = get_int_b2(bxi->qs, kqsx);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bxi->qh[kqsx];
+
+ #pragma unroll
+ for (int l = 0; l < QR1_S/2; ++l) {
+ const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)];
+
+ const int grid0 = (grid >> 0) & 0x0F0F0F0F;
+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid0;
+ x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid1;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1);
+ const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_ds[i*sram_stride + kqsx] = make_half2(d1q, d1q*delta);
+#else
+ x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XXS)) / 2;
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride;
+
+ const int q2 = get_int_b2(bxi->qs, 2*kqsx+0);
+ const uint8_t * aux8 = (const uint8_t *) &q2;
+ const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1);
+
+#pragma unroll
+ for (int l = 0; l < QR2_XXS; ++l) {
+ const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[l]];
+ const uint32_t signs = unpack_ksigns(aux32 >> (7 * l));
+
+ const int signs0 = __vcmpne4(signs & 0x08040201, 0);
+ const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0);
+
+ const int signs1 = __vcmpne4(signs & 0x80402010, 0);
+ const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0;
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const int ls = aux32 >> 27 | 1; // (scale * 2 + 1)
+ const float d = bxi->d;
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4
+#else
+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XS, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XS)) / 2;
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride;
+
+ const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));
+ const uint16_t * q2 = (const uint16_t *) &q2_packed;
+
+ #pragma unroll
+ for (int l = 0; l < QR2_XS; ++l) {
+ const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l] & 0x1FF];
+ const uint32_t signs = unpack_ksigns(q2[l] >> 9);
+
+ const int signs0 = __vcmpne4(signs & 0x08040201, 0);
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+
+ const int signs1 = __vcmpne4(signs & 0x80402010, 0);
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const int ls = bxi->scales[kqsx];
+ const float d = bxi->d;
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;
+ x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;
+#else
+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;
+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_s(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_S)) / 2;
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride;
+
+ const int qs_packed = get_int_b2(bxi->qs, kqsx);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bxi->qh[kqsx];
+
+ const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx);
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+
+#pragma unroll
+ for (int l = 0; l < QR2_S; ++l) {
+ const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300)));
+
+ const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);
+ const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);
+
+ const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);
+ const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const int ls = bxi->scales[kqsx];
+ const float d = bxi->d;
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;
+ x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;
+#else
+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;
+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_xxs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_XXS)) / 2;
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride;
+
+ const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));
+ const uint8_t * q3 = (const uint8_t *) &q3_packed;
+ const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx);
+
+#pragma unroll
+ for (int l = 0; l < QR3_XXS; ++l) {
+ const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]);
+ const uint32_t signs = unpack_ksigns(aux32 >> (7*l));
+
+ const int signs0 = __vcmpne4(signs & 0x08040201, 0);
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+
+ const int signs1 = __vcmpne4(signs & 0x80402010, 0);
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const int ls = aux32 >> 28;
+ const float d = bxi->d;
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + kqsx] = (ls*d + d/2)/2;
+#else
+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = (ls*d + d/2)/2;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_s(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_S)) / 2;
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {
+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride;
+
+ const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bxi->qh[kqsx];
+
+ const int signs_packed_32 = get_int_b2(bxi->signs, kqsx);
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+
+#pragma unroll
+ for (int l = 0; l < QR3_S; ++l) {
+ const int2 grid_pos = make_int2(
+ iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)],
+ iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]);
+
+ const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);
+ const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);
+
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid_l;
+ x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid_h;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid_l;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid_h;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F);
+ const float d = bxi->d;
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + kqsx] = ls*d;
+#else
+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = ls*d;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_XS);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int kqsx = threadIdx.x % threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;
+
+ const int aux_q4 = get_int_b4(bxi->qs, kqsx);
+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);
+ const int k0 = 8 * (kqsx / 4) + kqsx % 4;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + k0 + 0] = v.x;
+ x_qs[i*sram_stride + k0 + 4] = v.y;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 4] = v.y;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ constexpr int rows_per_warp = warp_size / 8;
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {
+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / (MMQ_TILE_NE_K/4);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;
+
+ const float d = __half2float(bxi->d);
+
+ const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F)
+ | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + threadIdx.x % 8] = d * (ls - 32);
+#else
+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + threadIdx.x % 8] = d * (ls - 32);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_nl(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+ const int kbx = txi / QI4_NL;
+ const int kqsx = txi % QI4_NL;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx;
+
+ const int aux_q4 = get_int_b2(bxi->qs, kqsx);
+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);
+ const int k0 = kbx * (2 * QI4_NL) + kqsx;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + k0 + 0] = v.x;
+ x_qs[i*sram_stride + k0 + QI4_NL] = v.y;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL;
+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {
+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + kbxd] = __half2float(bxi->d);
+#else
+ x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+// ---------------------------------------------------------------------------------------------
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_MXFP4, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR_MXFP4);
+ constexpr int nrows = warp_size / threads_per_row;
+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;
+ const int kbx = txi / QI_MXFP4;
+ const int kqsx = txi % QI_MXFP4;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {
+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbx;
+
+ const int aux_q4 = get_int_b1(bxi->qs, kqsx);
+ const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);
+ const int k0 = kbx * (2 * QI_MXFP4) + kqsx;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + k0 + 0] = v.x;
+ x_qs[i*sram_stride + k0 + QI_MXFP4] = v.y;
+#else
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;
+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI_MXFP4] = v.y;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI_MXFP4;
+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;
+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {
+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;
+
+ if (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbxd;
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_df[i*sram_stride + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;
+#else
+ x_df[i*(MMQ_TILE_NE_K/QI_MXFP4) + i/QI_MXFP4 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4_fp4(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4);
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4);
+
+ int * x_qs = (int *) x_tile;
+ uint32_t * x_sc = (uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);
+
+ const int txi = threadIdx.x;
+
+ constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback, GGML_PREC_Q4);
+
+ constexpr int threads_per_row = iter_k / QK_MXFP4; // each thread processes 1 block
+ constexpr int rows_per_warp = warp_size / threads_per_row;
+ const int kbx = txi % threads_per_row;
+ const int row_in_warp = txi / threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {
+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;
+
+ if constexpr (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i * stride + kbx;
+
+ // quantize_mxfp4_mmq permutes nibbles to match the quantized format
+ const int k0 = kbx * 4;
+ memcpy(x_qs + i*sram_stride + k0, bxi->qs, 16);
+
+ // Load E8M0 scales: pack 2 consecutive scales into one uint32
+ if (kbx % 2 == 0) {
+ uint32_t e = bxi->e;
+ e |= ((bxi + 1)->e << 8);
+ x_sc[i*sram_stride + kbx / 2] = e;
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);
+#else
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_NVFP4, I);
+ int * x_qs = (int *) x_tile;
+ float * x_df = (float *) (x_qs + txs.qs);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int threads_per_row = MMQ_ITER_K / QK_NVFP4;
+ constexpr int rows_per_warp = warp_size / threads_per_row;
+ const int kbx = threadIdx.x % threads_per_row;
+ const int row_in_warp = threadIdx.x / threads_per_row;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {
+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;
+
+ if constexpr (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx;
+ const uint32_t * __restrict__ src_qs = reinterpret_cast<const uint32_t *>(bxi->qs);
+ const int kqs = 16 * kbx;
+ const int ksc = 4 * kbx;
+
+#pragma unroll
+ for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {
+ const int2 q0 = get_int_from_table_16(src_qs[2 * sub + 0], kvalues_mxfp4);
+ const int2 q1 = get_int_from_table_16(src_qs[2 * sub + 1], kvalues_mxfp4);
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ x_qs[i*sram_stride + kqs + 4 * sub + 0] = q0.x;
+ x_qs[i*sram_stride + kqs + 4 * sub + 1] = q1.x;
+ x_qs[i*sram_stride + kqs + 4 * sub + 2] = q0.y;
+ x_qs[i*sram_stride + kqs + 4 * sub + 3] = q1.y;
+ x_df[i*sram_stride + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]);
+#else
+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 0] = q0.x;
+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 1] = q1.x;
+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 2] = q0.y;
+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 3] = q1.y;
+ x_df[i * (2 * MMQ_TILE_NE_K * 2 / QI_NVFP4) + i / (QK_NVFP4_SUB / QI_NVFP4) + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]);
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4(
+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4);
+ constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback, GGML_PREC_Q4);
+ constexpr int threads_per_row = iter_k / QK_NVFP4; // each thread processes 1 block
+ constexpr int rows_per_warp = warp_size / threads_per_row;
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4);
+
+ uint32_t * x_u32 = (uint32_t *) x_tile;
+
+ const int txi = threadIdx.x;
+ const int kbx = txi % threads_per_row;
+ const int row_in_warp = txi / threads_per_row;
+
+ const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx;
+ uint32_t * x_u32_scale = x_u32 + 64 + kbx;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {
+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;
+
+ if constexpr (fallback) {
+ i = min(i, i_max);
+ }
+
+ const block_nvfp4 * bxi = bxi_base + i * stride;
+
+ const uint32_t * src_qs = reinterpret_cast<const uint32_t *>(bxi->qs);
+
+#pragma unroll
+ for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {
+ x_u32[i*sram_stride + 8*kbx + 2 * sub + 0] = src_qs[2 * sub + 0];
+ x_u32[i*sram_stride + 8*kbx + 2 * sub + 1] = src_qs[2 * sub + 1];
+ }
+
+ x_u32_scale[i*sram_stride] = get_int_b4(bxi->d, 0);
+ }
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-vec-dot.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-vec-dot.cuh
new file mode 100644
index 00000000..a75dd046
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-vec-dot.cuh
@@ -0,0 +1,1237 @@
+#pragma once
+
+#include "vecdotq.cuh"
+#include "mma.cuh"
+
+using namespace ggml_cuda_mma;
+
+#include "mmq.cuh"
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+ const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);
+
+ int u[2*VDR_Q4_0_Q8_1_MMQ];
+
+ constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes();
+ constexpr int mcpy_int = max_cpy / sizeof(int);
+ static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ");
+
+ int tmp0[4], tmp1[4];
+
+ #pragma unroll
+ for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) {
+ ggml_cuda_memcpy_1<max_cpy>(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] );
+ ggml_cuda_memcpy_1<max_cpy>(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_0 + l0 * mcpy_int]);
+ }
+
+ u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3];
+ u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3];
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>
+ (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_0], u,
+ x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+ const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);
+
+ int u[2*VDR_Q4_1_Q8_1_MMQ];
+
+ constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes();
+ constexpr int mcpy_int = max_cpy / sizeof(int);
+ static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ");
+
+ int tmp0[4], tmp1[4];
+
+ #pragma unroll
+ for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) {
+ ggml_cuda_memcpy_1<max_cpy>(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] );
+ ggml_cuda_memcpy_1<max_cpy>(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_1 + l0 * mcpy_int]);
+ }
+
+ u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3];
+ u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3];
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>
+ (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_1], u,
+ x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMQ>
+ (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % MMQ_TILE_NE_K],
+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (MMQ_TILE_NE_K/QI8_1)]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback, mmq_q8_1_ds_layout ds_layout>
+static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr data_layout input_layout = get_input_data_layout();
+ typedef tile<16, 8, int, input_layout> tile_A;
+ typedef tile<16, 8, int, input_layout> tile_B;
+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+ const half2 * y_ds = (const half2 *) y;
+
+ const int i0 = (threadIdx.y / ntx) * rows_per_warp;
+
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {
+ const int k0 = k00 + k01;
+
+ tile_A A[ntx];
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+ tile_B B;
+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);
+
+ float dB;
+ const int j = j0 + tile_C::get_j(0);
+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {
+ dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+ } else {
+ dB = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C C;
+ mma(C, A[n], B);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ const int i = i0 + n*tile_A::I + tile_C::get_i(l);
+ const float dA = x_df[i*sram_stride + k0/QI8_0];
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA*dB;
+ }
+ }
+ }
+ }
+#else
+ typedef tile<16, 8, int> tile_A;
+ typedef tile< 8, 8, int> tile_B;
+ typedef tile<16, 8, int> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+ const half2 * y_ds = (const half2 *) y;
+
+ tile_A A[ntx][MMQ_TILE_NE_K/QI8_0];
+ float dA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_0];
+
+ const int i0 = (threadIdx.y/ntx)*rows_per_warp;
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {
+ const int k0 = k00 + k01;
+
+ load_ldmatrix(A[n][k01/QI8_0], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {
+ const int k0 = k00 + k01;
+
+ dA[n][l][k01/QI8_0] = x_df[i*sram_stride + k0/QI8_0];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {
+ tile_B B;
+ float dB[tile_C::ne/2];
+
+ load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int j = j0 + tile_C::get_j(l);
+
+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {
+ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+ } else {
+ dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C C;
+ mma(C, A[n][k01/QI8_0], B);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2];
+ }
+ }
+ }
+ }
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+}
+
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>
+ (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],
+ x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr data_layout input_layout = get_input_data_layout();
+ typedef tile<16, 8, int, input_layout> tile_A;
+ typedef tile<16, 8, int, input_layout> tile_B;
+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_dm = (const half2 *) y;
+
+ const int i0 = (threadIdx.y / ntx) * rows_per_warp;
+
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {
+ const int k0 = k00 + k01;
+
+ tile_A A[ntx];
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+ tile_B B;
+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);
+
+ const int j = j0 + tile_C::get_j(0);
+ const float2 dsB = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C C;
+ mma(C, A[n], B);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ const int i = i0 + n*tile_A::I + tile_C::get_i(l);
+ float2 dmA = __half22float2(x_dm[i*sram_stride + k0/QI8_1]);
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.x*dsB.x*C.x[l];
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.y*dsB.y;
+ }
+ }
+ }
+ }
+#else
+ typedef tile<16, 8, int> tile_A;
+ typedef tile< 8, 8, int> tile_B;
+ typedef tile<16, 8, int> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_dm = (const half2 *) y;
+
+ tile_A A[ntx][MMQ_TILE_NE_K/QI8_1];
+ float2 dmA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_1];
+
+ const int i0 = (threadIdx.y/ntx)*rows_per_warp;
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {
+ const int k0 = k00 + k01;
+
+ load_ldmatrix(A[n][k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {
+ const int k0 = k00 + k01;
+
+ dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*sram_stride + k0/QI8_1]);
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {
+ tile_B B;
+ float2 dsB[tile_C::ne/2];
+
+ load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int j = j0 + tile_C::get_j(l);
+
+ dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C C;
+ mma(C, A[n][k01/QI8_1], B);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l];
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y;
+ }
+ }
+ }
+ }
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+}
+
+// Used for NVFP4, Q3_K, IQ2_S, and IQ2_XS
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, I);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_16_q8_1_impl<QI8_0>(
+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0],
+ &y_qs[j*MMQ_TILE_Y_K + k01],
+ &x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)],
+ y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+// Used for Q3_K, IQ2_S, and IQ2_XS:
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr data_layout input_layout = get_input_data_layout();
+ typedef tile<16, 4, int, input_layout> tile_A;
+ typedef tile<16, 4, int, input_layout> tile_B;
+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+ const int i0 = (threadIdx.y / ntx) * rows_per_warp;
+
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {
+ const int k0 = k00 + k01;
+
+ tile_A A[ntx];
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+ tile_B B;
+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);
+
+ const int j = j0 + tile_C::get_j(0);
+ const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C C;
+ mma(C, A[n], B);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*sram_stride + k0/4] * dB;
+ }
+ }
+ }
+ }
+#elif defined(TURING_MMA_AVAILABLE)
+
+ typedef tile<16, 4, int> tile_A;
+ typedef tile<16, 8, int> tile_A_8;
+ typedef tile< 8, 4, int> tile_B;
+ typedef tile<16, 8, int> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);
+
+ tile_A A[ntx][8];
+ float dA[ntx][tile_C::ne/2][8];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {
+ const int k0 = k00 + k01;
+
+ load_ldmatrix(((tile_A_8 *) A[n])[k01/8], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {
+ const int k0 = k00 + k01;
+
+ dA[n][l][k01/4] = x_df[i*sram_stride + k0/4];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {
+ tile_B B[2];
+ float dB[tile_C::ne/2];
+
+ // Here load_generic is faster than load_ldmatrix.
+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);
+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int j = j0 + tile_C::get_j(l);
+
+ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C C[2];
+ mma(C[0], A[n][k01/4 + 0], B[0]);
+ mma(C[1], A[n][k01/4 + 1], B[1]);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]);
+ }
+ }
+ }
+ }
+#else
+ GGML_UNUSED_VARS(x, y, sum, k00);
+ NO_DEVICE_CODE;
+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+ float2 y_df[J/nwarps];
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);
+ }
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K/2; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ constexpr int ns = 2;
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq<ns>(
+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],
+ &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,
+ &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);
+ }
+ }
+ }
+
+ // Some compilers fail to unroll the loop over k01 if there is a conditional statement for ns in the inner loop.
+ // As a workaround 2 separate loops are used instead.
+#pragma unroll
+ for (int k01 = MMQ_TILE_NE_K/2; k01 < MMQ_TILE_NE_K; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ constexpr int ns = 1;
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq<ns>(
+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],
+ &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,
+ &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr data_layout input_layout = get_input_data_layout();
+ typedef tile<16, 4, int, input_layout> tile_A;
+ typedef tile<16, 4, int, input_layout> tile_B;
+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+ const int i0 = (threadIdx.y / ntx) * rows_per_warp;
+
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {
+ const int k0 = k00 + k01;
+
+ tile_A A[ntx];
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+ tile_B B;
+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);
+
+ const int j = j0 + tile_C::get_j(0);
+ const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y;
+ const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0
+ : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y
+ : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x);
+
+ tile_C Cm;
+ if (k01 >= MMQ_TILE_NE_K * 3/4) {
+ tile_A A1;
+#pragma unroll
+ for (int l = 0; l < tile_A::ne; ++l) {
+ A1.x[l] = 0x01010101;
+ }
+ mma(Cm, A1, B);
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C Cd;
+ mma(Cd, A[n], B);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);
+ const float2 dm = __half22float2(x_dm[i*sram_stride + k0/4]);
+ float tmp = Cd.x[l]*dm.x;
+ if (k01 >= MMQ_TILE_NE_K * 3/4) {
+ tmp -= Cm.x[l]*dm.y;
+ }
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB;
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB;
+ }
+ }
+ }
+ }
+#elif defined(TURING_MMA_AVAILABLE)
+
+ typedef tile<16, 4, int> tile_A;
+ typedef tile<16, 8, int> tile_A_8;
+ typedef tile< 8, 4, int> tile_B;
+ typedef tile<16, 8, int> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);
+
+ tile_A A[ntx][8];
+ float dA[ntx][tile_C::ne/2][8];
+ float mA[ntx][tile_C::ne/2][8];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {
+ const int k0 = k00 + k01;
+
+ load_ldmatrix(((tile_A_8 *) A[n])[k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1/2) {
+ const int k0 = k00 + k01;
+
+ const float2 dm = __half22float2(x_dm[i*sram_stride + k0/(QI8_1/2)]);
+
+ dA[n][l][k01/(QI8_1/2)] = dm.x;
+ mA[n][l][k01/(QI8_1/2)] = dm.y;
+ }
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+ float2 dB[tile_C::ne/2];
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int j = j0 + tile_C::get_j(l);
+
+ dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);
+ }
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {
+ tile_B B[2];
+
+ // Here load_generic is faster than load_ldmatrix.
+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);
+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);
+
+ tile_C Cm[2];
+ if (k01 >= MMQ_TILE_NE_K * 3/4) {
+ tile_A A1;
+ A1.x[0] = 0x01010101;
+ A1.x[1] = 0x01010101;
+ mma(Cm[0], A1, B[0]);
+ mma(Cm[1], A1, B[1]);
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C Cd[2];
+
+ mma(Cd[0], A[n][k01/4 + 0], B[0]);
+ mma(Cd[1], A[n][k01/4 + 1], B[1]);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1];
+ if (k01 >= MMQ_TILE_NE_K * 3/4) {
+ tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1];
+ }
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*(k01 < MMQ_TILE_NE_K/2 ? dB[l%2].x : dB[l%2].y);
+ }
+ }
+ }
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K * 3/4; k01 += QI8_1) {
+ float2 sB[tile_C::ne/2];
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int j = j0 + tile_C::get_j(l);
+
+ sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x;
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y;
+ }
+ }
+ }
+ }
+#else
+ GGML_UNUSED_VARS(x, y, sum, k00);
+ NO_DEVICE_CODE;
+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * x_sc = (const int *) x_df + txs.dm;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ const int8_t * scales = ((const int8_t *) (x_sc + i*(MMQ_TILE_NE_K/8) + i/8)) + k0/4;
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q3_K_q8_1_impl_mmq(
+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales,
+ x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * x_sc = (const int *) x_dm + txs.dm;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ const uint8_t * sc = (const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/32] + 2*(k01/16);
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_K_q8_1_impl_mmq(
+ &x_qs[i*(MMQ_TILE_NE_K + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,
+ x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I);
+ const int * x_qs = (const int *) x;
+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;
+ const int * x_sc = (const int *) x_dm + txs.dm;
+ const int * y_qs = (const int *) y + 4;
+ const half2 * y_ds = (const half2 *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k00/32]) + 2*(k01/16);
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q5_K_q8_1_impl_mmq(
+ &x_qs[i*(QR5_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,
+ x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
+
+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I);
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + txs.qs;
+ const int * x_sc = (const int *) x_df + txs.dm;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+// #pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ const int8_t * sc = ((const int8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/16]);
+
+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q6_K_q8_1_impl_mmq(
+ &x_qs[i*(QR6_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc,
+ x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);
+ }
+ }
+ }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ constexpr data_layout input_layout = get_input_data_layout();
+ typedef tile<16, 4, int, input_layout> tile_A;
+ typedef tile<16, 4, int, input_layout> tile_B;
+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;
+ const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+ const int i0 = (threadIdx.y / ntx) * rows_per_warp;
+
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {
+ const int k0 = k00 + k01;
+
+ tile_A A[ntx];
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+ tile_B B;
+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);
+
+ const int j = j0 + tile_C::get_j(0);
+ const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C C;
+ mma(C, A[n], B);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);
+ const int8_t * sc = (const int8_t *) (x_sc + i*sram_stride + k00/16);
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*sram_stride] * dB;
+ }
+ }
+ }
+ }
+#elif defined(TURING_MMA_AVAILABLE)
+
+ typedef tile<16, 4, int> tile_A;
+ typedef tile< 8, 4, int> tile_B;
+ typedef tile<16, 8, int> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;
+ const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;
+ const int * y_qs = (const int *) y + 4;
+ const float * y_df = (const float *) y;
+
+ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);
+
+ tile_A A[ntx][8];
+ int scA[ntx][tile_C::ne/2][8];
+ float dA[ntx][tile_C::ne/2];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {
+ const int k0 = k00 + k01;
+
+ load_ldmatrix(A[n][k01/4 + 0], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + 0), sram_stride);
+ load_ldmatrix(A[n][k01/4 + 1], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + tile_A::J), sram_stride);
+ }
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 16) {
+ const int k0 = k00 + k01;
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);
+
+ const int sc_packed = x_sc[i*sram_stride + k0/16];
+ const int8_t * sc = (const int8_t *) &sc_packed;
+
+#pragma unroll
+ for (int ksc = 0; ksc < sizeof(int); ++ksc) {
+ scA[n][l][k01/4 + ksc] = sc[ksc];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);
+
+ dA[n][l] = x_df[i*sram_stride];
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+ float tmp[ntx][tile_C::ne] = {{0.0f}};
+
+#pragma unroll
+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {
+ tile_B B[2];
+ float dB[tile_C::ne/2];
+
+ // Here load_generic is faster than load_ldmatrix.
+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K);
+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + tile_B::J + k01, MMQ_TILE_Y_K);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne/2; ++l) {
+ const int j = j0 + tile_C::get_j(l);
+
+ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ tile_C C[2];
+ mma(C[0], A[n][k01/4 + 0], B[0]);
+ mma(C[1], A[n][k01/4 + 1], B[1]);
+
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2];
+ }
+ }
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp[n][l]*dA[n][l/2];
+ }
+ }
+ }
+#else
+ GGML_UNUSED_VARS(x, y, sum, k00);
+ NO_DEVICE_CODE;
+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE
+}
+
+// ---------------------------------------------------------------------------------------------
+
+// Shared MMA kernel for MXFP4 and NVFP4 on Blackwell.
+// Both quantizations encode values as e2m1 (FP4) and produce one uint32 scale per
+// m16n8k64 MMA call; only the PTX kind (scale_vec::2X ue8m0 vs scale_vec::4X ue4m3)
+// and the per-type stride constant differ.
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_fp4_fp4_mma(
+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {
+
+ typedef tile<16, 8, int> tile_A;
+ typedef tile<8, 8, int> tile_B;
+ typedef tile<16, 8, float> tile_C;
+
+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4);
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q4);
+ constexpr int ntx = rows_per_warp / tile_C::I;
+ constexpr int nfrags = MMQ_TILE_NE_K / tile_A::J;
+
+ y += (threadIdx.y % ntx) * (tile_C::J * MMQ_TILE_Y_K);
+
+ const int * x_qs = (const int *) x;
+ const uint32_t * x_sc = (const uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);
+ const int * y_qs = (const int *) y + 4;
+ const uint32_t * y_sc = (const uint32_t *) y;
+
+ // 2 threads per quad supply the packed scale register to the block_scale MMA,
+ // see https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-block-scaling
+ const int tidx_A = threadIdx.x / 4 + (threadIdx.x % 2) * 8;
+ const int tidx_B = threadIdx.x / 4;
+ const int i0 = (threadIdx.y / ntx) * rows_per_warp;
+
+ tile_A A[ntx][nfrags];
+ uint32_t scaleA[ntx][nfrags];
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int frag = 0; frag < nfrags; ++frag) {
+ const int k0 = k00 + frag * tile_A::J;
+ load_ldmatrix(A[n][frag], x_qs + (i0 + n * tile_A::I) * sram_stride + k0, sram_stride);
+ scaleA[n][frag] = x_sc[(i0 + n * tile_A::I + tidx_A) * sram_stride + k0 / tile_A::J];
+ }
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx * tile_C::J) {
+ tile_B B[nfrags];
+ uint32_t scaleB[nfrags];
+
+#pragma unroll
+ for (int frag = 0; frag < nfrags; ++frag) {
+ const int k0 = frag * tile_B::J;
+ load_generic(B[frag], y_qs + j0 * MMQ_TILE_Y_K + k0, MMQ_TILE_Y_K);
+ scaleB[frag] = y_sc[(j0 + tidx_B) * MMQ_TILE_Y_K + frag];
+ }
+
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+ // accumulate in place into the output sum array
+ tile_C & C = *reinterpret_cast<tile_C *>(sum + (j0 / tile_C::J + n) * tile_C::ne);
+#pragma unroll
+ for (int frag = 0; frag < nfrags; ++frag) {
+ mma_block_scaled_fp4<type>(C, A[n][frag], B[frag], scaleA[n][frag], scaleB[frag]);
+ }
+ }
+ }
+}
+
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu
new file mode 100644
index 00000000..f754837d
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu
@@ -0,0 +1,522 @@
+#include "common.cuh"
+#include "mmq.cuh"
+#include "quantize.cuh"
+#include "mmid.cuh"
+
+#include <cstdint>
+
+static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream, const ggml_prec prec_src1) {
+ switch (args.type_x) {
+ case GGML_TYPE_Q1_0:
+ mul_mat_q_case<GGML_TYPE_Q1_0, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q2_0:
+ mul_mat_q_case<GGML_TYPE_Q2_0, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q4_0:
+ mul_mat_q_case<GGML_TYPE_Q4_0, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q4_1:
+ mul_mat_q_case<GGML_TYPE_Q4_1, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q5_0:
+ mul_mat_q_case<GGML_TYPE_Q5_0, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q5_1:
+ mul_mat_q_case<GGML_TYPE_Q5_1, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q8_0:
+ mul_mat_q_case<GGML_TYPE_Q8_0, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+// -----------------------------------------------------------------------
+ case GGML_TYPE_Q2_K:
+ mul_mat_q_case<GGML_TYPE_Q2_K, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q3_K:
+ mul_mat_q_case<GGML_TYPE_Q3_K, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q4_K:
+ mul_mat_q_case<GGML_TYPE_Q4_K, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q5_K:
+ mul_mat_q_case<GGML_TYPE_Q5_K, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_Q6_K:
+ mul_mat_q_case<GGML_TYPE_Q6_K, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+// -----------------------------------------------------------------------
+ case GGML_TYPE_IQ1_S:
+ mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ2_XXS:
+ mul_mat_q_case<GGML_TYPE_IQ2_XXS, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ2_XS:
+ mul_mat_q_case<GGML_TYPE_IQ2_XS, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ2_S:
+ mul_mat_q_case<GGML_TYPE_IQ2_S, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ3_XXS:
+ mul_mat_q_case<GGML_TYPE_IQ3_XXS, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ3_S:
+ mul_mat_q_case<GGML_TYPE_IQ3_S, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ4_XS:
+ mul_mat_q_case<GGML_TYPE_IQ4_XS, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_IQ4_NL:
+ mul_mat_q_case<GGML_TYPE_IQ4_NL, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+// -----------------------------------------------------------------------
+ case GGML_TYPE_MXFP4:
+ // src1 at Q4 uses the native FP4 instructions, which are Blackwell-only
+ if (prec_src1 == GGML_PREC_Q4) {
+ mul_mat_q_case<GGML_TYPE_MXFP4, GGML_PREC_Q4>(ctx, args, stream);
+ break;
+ }
+ mul_mat_q_case<GGML_TYPE_MXFP4, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ case GGML_TYPE_NVFP4:
+ if (prec_src1 == GGML_PREC_Q4) {
+ mul_mat_q_case<GGML_TYPE_NVFP4, GGML_PREC_Q4>(ctx, args, stream);
+ break;
+ }
+ mul_mat_q_case<GGML_TYPE_NVFP4, GGML_PREC_Q8>(ctx, args, stream);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+// overrides the src1 precision requested by the graph, "auto" keeps the requested one
+static ggml_prec ggml_cuda_mmq_get_prec_env() {
+ const char * env_c = getenv("GGML_CUDA_MMQ_PREC");
+ if (env_c == nullptr) {
+ return GGML_PREC_UNDEFINED;
+ }
+ std::string env_cpp = env_c;
+ for (char & c : env_cpp) {
+ c = std::tolower(c);
+ }
+ if (env_cpp == "q4") {
+ return GGML_PREC_Q4;
+ }
+ if (env_cpp == "q8") {
+ return GGML_PREC_Q8;
+ }
+ if (env_cpp != "auto") {
+ GGML_LOG_WARN("%s: Unknown value for GGML_CUDA_MMQ_PREC: '%s'. Available: 'q4', 'q8', 'auto'.\n", __func__, env_cpp.c_str());
+ }
+ return GGML_PREC_UNDEFINED;
+}
+
+// src1 is quantized to Q8_1 unless the FP4 types can use 4-bit activations, in which case they
+// default to the native W4A4 instructions on Blackwell.
+static ggml_prec ggml_cuda_mmq_get_prec_src1(const ggml_tensor * src0, const ggml_tensor * dst, const int cc) {
+ static const ggml_prec prec_env = ggml_cuda_mmq_get_prec_env();
+
+ ggml_prec prec = prec_env;
+ if (prec == GGML_PREC_UNDEFINED) {
+ prec = (ggml_prec) ggml_get_op_params_i32(dst, 3);
+ }
+
+ // Q4 only for the FP4 types on Blackwell
+ GGML_ASSERT(prec == GGML_PREC_UNDEFINED || prec == GGML_PREC_Q8 || prec == GGML_PREC_Q4);
+ const bool can_use_q4 = (src0->type == GGML_TYPE_NVFP4 || src0->type == GGML_TYPE_MXFP4) && blackwell_mma_available(cc);
+ if (prec == GGML_PREC_Q8 || !can_use_q4) {
+ return GGML_PREC_Q8;
+ }
+ return GGML_PREC_Q4;
+}
+
+void ggml_cuda_mul_mat_q(
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
+ int64_t ncols_max) {
+ // FreeToken: bf16 and fp16 activations are quantized directly, except for the native fp4 paths; dst is fp32, bf16 or fp16
+ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);
+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+
+ cudaStream_t stream = ctx.stream();
+
+ const int id = ggml_cuda_get_device();
+ const int cc = ggml_cuda_info().devices[id].cc;
+ const size_t smpbo = ggml_cuda_info().devices[id].smpbo;
+
+ const size_t ts_src0 = ggml_type_size(src0->type);
+ const size_t ts_src1 = ggml_type_size(src1->type);
+ const size_t ts_dst = ggml_type_size(dst->type);
+
+ GGML_ASSERT( nb00 == ts_src0);
+ GGML_ASSERT( nb10 == ts_src1);
+ GGML_ASSERT( nb0 == ts_dst);
+ GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));
+
+ const char * src0_d = (const char *) src0->data;
+ const float * src1_d = (const float *) src1->data;
+ const mmq_dst dst_d = {dst->data, dst->type};
+ const nv_bfloat16 * src1_bf16 = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1->data : nullptr;
+ const half * src1_f16 = src1->type == GGML_TYPE_F16 ? (const half *) src1->data : nullptr;
+
+ // If src0 is a temporary compute buffer, clear any potential padding.
+ if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
+ const size_t size_data = ggml_nbytes(src0);
+ const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);
+ if (size_alloc > size_data) {
+ GGML_ASSERT(ggml_is_contiguously_allocated(src0));
+ GGML_ASSERT(!src0->view_src);
+ CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));
+ }
+ }
+
+ const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);
+
+ const int64_t s01 = src0->nb[1] / ts_src0;
+ const int64_t s1 = dst->nb[1] / ts_dst;
+ const int64_t s02 = src0->nb[2] / ts_src0;
+ const int64_t s2 = dst->nb[2] / ts_dst;
+ const int64_t s03 = src0->nb[3] / ts_src0;
+ const int64_t s3 = dst->nb[3] / ts_dst;
+
+ const bool fallback = ggml_cuda_mmq_needs_fallback(ne01);
+
+ const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc);
+
+ const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;
+ GGML_ASSERT(!use_native_fp4 || src1->type == GGML_TYPE_F32);
+ const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);
+ const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ;
+
+ int J_best = 0;
+ int nthreads_best = 0;
+ {
+ int64_t ncols_opt = ne11;
+ if (ids) {
+ const int64_t n_expert_used = ids->ne[0];
+ // a flat grid runs only the tiles that hold tokens, so it sizes them for the average expert
+ ncols_opt = ncols_max > 0 ? ncols_max : (ne12*n_expert_used + ne02 - 1) / ne02;
+
+ // Each expert only sees ne12*n_expert_used/ne02 tokens on average.
+ // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.
+ if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) {
+ ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02;
+ }
+ }
+
+ int ntiles_J_best = INT_MAX;
+
+ for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) {
+ const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(src0->type, J, fallback, cc, prec_src1);
+ if (config.type == GGML_TYPE_COUNT) {
+ continue;
+ }
+
+ if (mmq_get_nbytes_shared(config, cc) > smpbo) {
+ continue;
+ }
+
+ const int ntiles_x = (ncols_opt + config.J - 1) / config.J;
+
+ if (ntiles_x < ntiles_J_best) {
+ J_best = J;
+ nthreads_best = config.nthreads;
+ ntiles_J_best = ntiles_x;
+ }
+ }
+ }
+ GGML_ASSERT(J_best > 0);
+
+ // A tile of size J can read in at most J - 1 extra columns.
+ // For simplicity, round up the padding of a full tile to a multiple of the number of bytes that nthreads can load in parallel.
+ const size_t src1_load_chunk_size = nthreads_best * sizeof(int);
+ const size_t src1_q8_1_padding = ((J_best * sizeof(block_q8_1_mmq) + src1_load_chunk_size - 1) / src1_load_chunk_size)
+ * src1_load_chunk_size;
+
+ if (!ids) {
+ const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;
+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);
+ ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());
+ if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) {
+ src1_scale.alloc(ne13*ne12*ne11);
+ }
+
+ {
+ const int64_t s11 = src1->nb[1] / ts_src1;
+ const int64_t s12 = src1->nb[2] / ts_src1;
+ const int64_t s13 = src1->nb[3] / ts_src1;
+ if (use_native_fp4) {
+ static constexpr size_t align_float8 = 32;
+ const bool use_aligned_float8 = ggml_cuda_is_aligned(src1, align_float8);
+ static_assert(sizeof(block_fp4_mmq) == 4 * sizeof(block_q8_1));
+ quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded,
+ ne11, ne12, ne13, stream);
+
+ } else if (src1_bf16) {
+ quantize_mmq_q8_1_cuda(src1_bf16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
+ ne11, ne12, ne13, stream);
+ } else if (src1_f16) {
+ quantize_mmq_q8_1_cuda(src1_f16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
+ ne11, ne12, ne13, stream);
+ } else {
+ quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
+ ne11, ne12, ne13, stream);
+ }
+ CUDA_CHECK(cudaGetLastError());
+ }
+
+ // Stride depends on quantization format
+ const int64_t s12 = use_native_fp4 ?
+ ne11 * ne10_padded * sizeof(block_fp4_mmq) / (QK_FP4_MMQ * sizeof(int)) :
+ ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));
+ const int64_t s13 = ne12*s12;
+
+ const mmq_args args = {
+ src0_d, src0->type, (const int *) src1_q8_1.ptr, nullptr, nullptr, dst_d,
+ src0->type == GGML_TYPE_NVFP4 && use_native_fp4 ? src1_scale.ptr : nullptr,
+ ne00, ne01, ne1, s01, ne11, s1,
+ ne02, ne12, s02, s12, s2,
+ ne03, ne13, s03, s13, s3,
+ ne1, J_best};
+ ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);
+ return;
+ }
+
+ GGML_ASSERT(ne13 == 1);
+ GGML_ASSERT(nb12 % nb11 == 0);
+ GGML_ASSERT(nb2 % nb1 == 0);
+
+ const int64_t n_expert_used = ids->ne[0];
+ const int64_t ne_get_rows = ne12 * n_expert_used;
+ GGML_ASSERT(ne1 == n_expert_used);
+
+ ggml_cuda_pool_alloc<int32_t> ids_src1(ctx.pool(), ne_get_rows);
+ ggml_cuda_pool_alloc<int32_t> ids_dst(ctx.pool(), ne_get_rows + J_best-1); // Needs to be padded for unconditional memory access.
+ ggml_cuda_pool_alloc<int32_t> expert_bounds(ctx.pool(), ne02 + 1);
+
+ // gate/up activations are broadcast across experts (ne11 == 1): quantize each token once and
+ // scatter to its slots. ids_src1 then holds the inverse map (token slot -> compact row).
+ const bool dedup_bcast = ne11 == 1 && n_expert_used > 1;
+
+ {
+ GGML_ASSERT(ids->nb[0] == ggml_element_size(ids));
+ const int si1 = ids->nb[1] / ggml_element_size(ids);
+ const int sis1 = nb12 / nb11;
+
+ ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(),
+ ne02, ne12, n_expert_used, ne11, si1, sis1, /*write_inverse =*/ dedup_bcast, stream);
+ CUDA_CHECK(cudaGetLastError());
+ }
+
+ // Every expert's column tiles fit in ntx_flat grid columns, so a flat grid needs no host sync to size it.
+ ggml_cuda_pool_alloc<int2> tile_map(ctx.pool());
+ int ntx_flat = 0;
+ if (ncols_max <= 0) {
+ const int jt_none = (ne_get_rows + J_best - 1) / J_best;
+ ntx_flat = jt_none + std::min<int64_t>(ne02, ne_get_rows);
+ tile_map.alloc(ntx_flat);
+ ggml_cuda_launch_mm_ids_tile_map(expert_bounds.get(), tile_map.get(), ne02, J_best, ntx_flat, jt_none, stream);
+ CUDA_CHECK(cudaGetLastError());
+ }
+
+ const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;
+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);
+ ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());
+ if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) {
+ src1_scale.alloc(ne12*n_expert_used);
+ }
+
+ const int64_t ne11_flat = ne12*n_expert_used;
+ const int64_t ne12_flat = 1;
+ const int64_t ne13_flat = 1;
+
+ {
+ const int64_t s11 = src1->nb[1] / ts_src1;
+ const int64_t s12 = src1->nb[2] / ts_src1;
+ const int64_t s13 = src1->nb[3] / ts_src1;
+
+ if (use_native_fp4) {
+ static constexpr size_t align_float8 = 32;
+ const bool use_aligned_float8 = ggml_cuda_is_aligned(src1, align_float8);
+ if (dedup_bcast) {
+ quantize_scatter_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10,
+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
+ } else {
+ quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13,
+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
+ }
+ } else if (dedup_bcast && src1_bf16) {
+ quantize_scatter_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
+ } else if (dedup_bcast && src1_f16) {
+ quantize_scatter_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
+ } else if (dedup_bcast) {
+ quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
+ } else if (src1_bf16) {
+ quantize_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
+ } else if (src1_f16) {
+ quantize_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
+ } else {
+ quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,
+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
+ }
+ CUDA_CHECK(cudaGetLastError());
+ }
+
+ static_assert(QK_FP4_MMQ == 8 * QK_MXFP4, "QK_FP4_MMQ needs to be 8 * QK_MXFP4");
+ const int64_t s12 = use_native_fp4 ? ne11 * ne10_padded * sizeof(block_fp4_mmq) / (QK_FP4_MMQ * sizeof(int)) :
+ ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));
+ const int64_t s13 = ne12*s12;
+
+ // Note that ne02 is used instead of ne12 because the number of y channels determines the z dimension of the CUDA grid.
+ const mmq_args args = {
+ src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), dst_d,
+ src1_scale.ptr,
+ ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,
+ ne02, ne02, s02, s12, s2,
+ ne03, ne13, s03, s13, s3,
+ ncols_max > 0 ? ncols_max : ne12, J_best, tile_map.ptr, ntx_flat};
+
+ ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);
+}
+
+bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) {
+#ifdef GGML_CUDA_FORCE_CUBLAS
+ return false;
+#endif // GGML_CUDA_FORCE_CUBLAS
+
+ bool mmq_supported;
+
+ switch (type) {
+ case GGML_TYPE_Q1_0:
+ case GGML_TYPE_Q2_0:
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+// -------------------------------------------------
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+// -------------------------------------------------
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ4_NL:
+// -------------------------------------------------
+ case GGML_TYPE_MXFP4:
+ case GGML_TYPE_NVFP4:
+ mmq_supported = true;
+ break;
+ default:
+ mmq_supported = false;
+ break;
+ }
+
+ if (!mmq_supported) {
+ return false;
+ }
+
+ // MMQ tiles require at least 48 KiB per-block shared memory; fall back to BLAS otherwise.
+ {
+ const int id = ggml_cuda_get_device();
+ const size_t smpbo = ggml_cuda_info().devices[id].smpbo;
+ if (smpbo < 48 * 1024) {
+ return false;
+ }
+ }
+
+ if (turing_mma_available(cc)) {
+ return true;
+ }
+
+ if (ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_DP4A) {
+ // for MoE, mmq is faster even without native dp4a
+ // TODO: check if cards older than pascal might benefit from this as well
+ return cc >= GGML_CUDA_CC_PASCAL && n_experts > 0;
+ }
+
+#ifdef GGML_CUDA_FORCE_MMQ
+ return true;
+#endif //GGML_CUDA_FORCE_MMQ
+
+ if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
+ return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;
+ }
+
+ if (amd_mfma_available(cc)) {
+ // As of ROCM 7.0 rocblas/tensile performs very poorly on CDNA3 and hipblaslt (via ROCBLAS_USE_HIPBLASLT)
+ // performs better but is currently suffering from a crash on this architecture.
+ // TODO: Revisit when hipblaslt is fixed on CDNA3
+ if (GGML_CUDA_CC_IS_CDNA3(cc)) {
+ return true;
+ }
+ if (n_experts > 64 || ne11 <= 128) {
+ return true;
+ }
+ if (type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) {
+ return true;
+ }
+ if (ne11 <= 256 && (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K)) {
+ return true;
+ }
+ return false;
+ }
+
+ if (amd_wmma_available(cc)) {
+ if (GGML_CUDA_CC_IS_RDNA3(cc)) {
+ // High expert counts are almost always better on MMQ due to
+ // the synchronization overhead in the cuBLAS/hipBLAS path:
+ // https://github.com/ggml-org/llama.cpp/pull/18202
+ if (n_experts >= 64) {
+ return true;
+ }
+
+ // For some quantization types MMQ can have lower peak TOPS than hipBLAS
+ // so it's only faster for sufficiently small batch sizes:
+ switch (type) {
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 128;
+ case GGML_TYPE_Q6_K:
+ return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ return GGML_CUDA_CC_IS_RDNA3_5(cc) || ne11 <= 128;
+ default:
+ return true;
+ }
+ }
+
+ // For RDNA4 MMQ is consistently faster than dequantization + hipBLAS:
+ // https://github.com/ggml-org/llama.cpp/pull/18537#issuecomment-3706422301
+ return true;
+ }
+
+ // gfx900 (Vega 10), gfx909, and gfx90c lack native dp4a, losing to dequant + hipBLAS
+ // for dense matrices; keep MMQ only for MoE, where the
+ // hipBLAS path is much slower.
+ if (cc == GGML_CUDA_CC_VEGA || GGML_CUDA_CC_IS_GCN_APU(cc)) {
+ return n_experts > 0;
+ }
+
+ // MUSA: the MMQ kernels compute wrong values on PH1 (MTT S5000).
+ if (cc == GGML_CUDA_CC_PH1) {
+ return false;
+ }
+
+ return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh
new file mode 100644
index 00000000..596a54ea
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh
@@ -0,0 +1,1661 @@
+#pragma once
+
+#include "common.cuh"
+
+#include <climits>
+#include <cstdint>
+
+#define MMQ_DP4A_MAX_BATCH_SIZE 64 // Max. batch size to use for dp4a MMQ kernels when FP16 tensor cores are available.
+#define MMQ_ITER_K 256
+#define MMQ_ITER_K_FP4 512
+#define MMQ_NWARPS 8
+
+// FreeToken: MMQ writes fp32, bf16 or fp16 results (dst_t); the sums stay fp32 until the store
+template <typename T> static __device__ __forceinline__ T mmq_cast(float v);
+template <> __device__ __forceinline__ float mmq_cast<float>(float v) { return v; }
+template <> __device__ __forceinline__ nv_bfloat16 mmq_cast<nv_bfloat16>(float v) { return __float2bfloat16(v); }
+template <> __device__ __forceinline__ half mmq_cast<half>(float v) { return __float2half(v); }
+
+struct mmq_dst {
+ void * p;
+ ggml_type type;
+};
+
+typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);
+typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);
+typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,
+ float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max);
+
+enum mmq_q8_1_ds_layout {
+ MMQ_Q8_1_DS_LAYOUT_D4,
+ MMQ_Q8_1_DS_LAYOUT_DS4,
+ MMQ_Q8_1_DS_LAYOUT_D2S6,
+};
+
+static constexpr int QK8_1_MMQ = 4*QK8_1;
+static constexpr int QK_FP4_MMQ = 2*QK8_1_MMQ;
+
+struct block_q8_1_mmq {
+ // The y float data is converted to a data layout that can simply be copied to shared memory as a contiguous block.
+ // The y float data is first grouped as blocks of 128 values.
+ // These blocks are then treated as individual data values and transposed.
+ //
+ // To avoid shared memory bank conflicts each block is padded with 16 bytes.
+ // This padding is also used to store block scales/partial sums.
+ // The scales multiplied with the quantized data are equal to the unquantized values.
+ // The partial sums are obtained by summing up a subgroup of the contained values (prior to quantization)
+ // and are only needed for performance reasons.
+ //
+ // The exact data stored depends on the x data type.
+ union {
+ float d4[4]; // 1 32 bit scale per 32 values, stored as d0,d1,d2,d3
+ half2 ds4[4]; // 1 16 bit scale + 1 16 bit partial sum per 32 values, stored as d0,s0,d1,s1,d2,s2,d3,s3
+ half d2s6[8]; // 1 16 bit scale per 64 values + 1 16 bit partial sum per 16 values for the first 96 values,
+ // stored as d0,d1,s1,s2,s3,s4,s5
+ };
+ int8_t qs[QK8_1_MMQ];
+};
+
+// this struct is used for fp4 data types (currently only used for Blackwell)
+// mxfp4 has block size 32, each int32 of d4 contains 2 e8m0 scales in the lower 16 bits
+// nvfp4 has block size 16, each int32 of d4 contains 4 ue4m3 scales
+struct block_fp4_mmq {
+ uint32_t d4[4];
+ int8_t qs[QK_FP4_MMQ / 2];
+};
+
+static_assert(sizeof(block_q8_1_mmq) == QK8_1_MMQ + 4*sizeof(half2), "Unexpected block_q8_1_mmq size");
+static_assert(sizeof(block_q8_1_mmq) == 4*sizeof(block_q8_1), "Unexpected block_q8_1_mmq size");
+static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected block_fp4_mmq size");
+
+static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {
+ switch (type_x) {
+ case GGML_TYPE_Q1_0:
+ case GGML_TYPE_Q2_0:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+ case GGML_TYPE_Q5_0:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_Q5_1:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+ case GGML_TYPE_Q8_0:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_MXFP4:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_NVFP4:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_Q2_K:
+ return MMQ_Q8_1_DS_LAYOUT_D2S6;
+ case GGML_TYPE_Q3_K:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ3_S:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ case GGML_TYPE_IQ1_S:
+ return MMQ_Q8_1_DS_LAYOUT_DS4;
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ4_NL:
+ return MMQ_Q8_1_DS_LAYOUT_D4;
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+struct tile_x_sizes {
+ int qs;
+ int dm;
+ int sc;
+};
+
+// Decouple shared memory tile sizes from WARP_SIZE to allow for different warp sizes.
+// The K dimension of the tiles has either,
+// 1*MMQ_TILE_NE_K==32 (always for TILE_Y_K) or 2*MMQ_TILE_NE_K==64 (typically for TILE_X_K),
+// 32 bit elements for the quantized data (does not include scales).
+// In other words, the size of the quantized data in the K dimension is a multiple of MMQ_TILE_NE_K.
+// The final tile size in K direction is padded to avoid shared memory bank conflicts,
+// in terms of 32 bit elements that means K % 2 == 1 for dp4a or K % 8 == 4 for mma.
+#define MMQ_TILE_NE_K 32
+
+// block_q8_1_mmq has (128 8-bit ints == 32 32-bit ints + 4 32-bit scales)
+#define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1)
+#define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K
+
+enum ggml_cuda_mmq_sram_layout {
+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0,
+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1,
+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K,
+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K,
+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K,
+ GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, // MXFP4 and NVFP4 on Blackwell.
+ GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, // Generic NVFP4
+};
+
+static constexpr __host__ __device__ int ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_sram_layout sram_layout) {
+ switch (sram_layout) {
+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0:
+ return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4;
+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1:
+ return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_1 + 4;
+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K:
+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4;
+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K:
+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4;
+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K:
+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7;
+ case GGML_CUDA_MMQ_SRAM_LAYOUT_FP4:
+ return 2*MMQ_TILE_NE_K + 8 + 4;
+ case GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4:
+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4;
+ default:
+ return -1;
+ }
+}
+
+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0) % 8 == 4, "Wrong padding.");
+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1) % 8 == 4, "Wrong padding.");
+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K) % 8 == 4, "Wrong padding.");
+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K) % 8 == 4, "Wrong padding.");
+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K) % 8 == 4, "Wrong padding.");
+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) % 8 == 4, "Wrong padding.");
+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4) % 8 == 4, "Wrong padding.");
+
+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) == ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1), "Wrong tile size for MXFP4");
+
+// Config options for the MMQ kernel.
+// Should not affect results, only speed/register pressure/shared memory use.
+struct ggml_cuda_mmq_config {
+ ggml_type type; // src0->type
+ int nthreads; // Number of threads per CUDA block.
+ int occupancy; // Targeted occupancy for the MMA kernel.
+ int I; // SRAM tile width in src0->ne[1]/dst->ne[0] direction.
+ int J; // SRAM tile width in src1->ne[1]/dst->ne[1] direction.
+ ggml_cuda_mmq_sram_layout sram_layout; // SRAM tile length in src0->ne[0]/src1->ne[0] direction (physical 32 bit elements).
+ int K_vram; // VRAM tile length in src0->ne[0]/src1->ne[0] direction (logical elements).
+ bool stream_k; // Whether or not to use stream-k decomposition.
+ bool fallback; // Whether a fallback for out-of-bounds check in src0->ne[1] direction is needed.
+
+ constexpr __host__ __device__ ggml_cuda_mmq_config(
+ ggml_type type, int nthreads, int occupancy, int I, int J, ggml_cuda_mmq_sram_layout sram_layout, int K_vram, bool stream_k, bool fallback) :
+ type(type), nthreads(nthreads), occupancy(occupancy), I(I), J(J), sram_layout(sram_layout), K_vram(K_vram), stream_k(stream_k), fallback(fallback) {}
+
+ constexpr __device__ int rows_per_warp() const {
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ return 16;
+#else
+ return J >= 48 && J % 16 == 0 ? 32 : 16;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ }
+
+ // TODO transition all combinations of GPUs and quantizations to the MMA data layout.
+ __host__ int use_mma_data_layout(const int cc) const {
+ if (amd_mfma_available(cc) || amd_wmma_available(cc) || turing_mma_available(cc)) {
+ return true;
+ }
+ return false;
+ }
+
+ constexpr __device__ bool use_mma_data_layout() const {
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)
+ return true;
+#else
+ return false;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)
+ }
+
+};
+
+#define CASE(type_, nthreads_, occupancy_, I_, J_, sram_layout_, K_vram_, stream_k_, fallback_) \
+ if (type == (type_) && J == (J_) && fallback == (fallback_)) { \
+ static_assert((nthreads_) % 32 == 0 && (nthreads_) <= 512, "bad nthreads"); \
+ static_assert( (occupancy_) <= 8, "bad occupancy"); \
+ static_assert((I_) % 32 == 0, "bad I"); \
+ static_assert((J_) % 8 == 0 && (J_) <= 128, "bad J"); \
+ static_assert((K_vram_) % 256 == 0, "bad K_vram"); \
+ return ggml_cuda_mmq_config((type_), (nthreads_), (occupancy_), (I_), (J_), (sram_layout_), (K_vram_), (stream_k_), (fallback_)); \
+ } \
+
+#include "mmq-config-pascal-older.cuh"
+#include "mmq-config-pascal-dp4a.cuh"
+#include "mmq-config-ampere.cuh"
+#include "mmq-config-blackwell.cuh"
+
+#include "mmq-config-gcn.cuh"
+#include "mmq-config-cdna.cuh"
+#include "mmq-config-rdna2.cuh"
+#include "mmq-config-rdna3.cuh"
+#include "mmq-config-rdna3-5.cuh"
+#include "mmq-config-rdna4.cuh"
+
+#undef CASE
+
+static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ if (GGML_CUDA_CC_IS_AMD(cc)) {
+ if (GGML_CUDA_CC_IS_GCN(cc)) {
+ return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
+ }
+ if (GGML_CUDA_CC_IS_CDNA(cc)) {
+ return ggml_cuda_mmq_get_config_cdna(type, J, fallback);
+ }
+ if (GGML_CUDA_CC_IS_RDNA4(cc)) {
+ return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
+ }
+ if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+ return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
+ }
+ if (GGML_CUDA_CC_IS_RDNA3(cc)) { // covers RDNA 3.0
+ return ggml_cuda_mmq_get_config_rdna3(type, J, fallback);
+ }
+ return ggml_cuda_mmq_get_config_rdna2(type, J, fallback);
+ }
+ if (blackwell_mma_available(cc)) {
+ // only src1 at Q4 uses the native FP4 config, higher precisions keep src1 at Q8_1
+ if (prec_src1 != GGML_PREC_Q4 && (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4)) {
+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
+ }
+ return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);
+ }
+ if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) {
+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
+ }
+ if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_DP4A) {
+ return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback);
+ }
+ return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback);
+}
+
+static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+#ifdef GGML_USE_HIP
+#ifdef GCN
+ return ggml_cuda_mmq_get_config_gcn(type, J, fallback);
+#elif defined(CDNA)
+ return ggml_cuda_mmq_get_config_cdna(type, J, fallback);
+#elif defined(RDNA4)
+ return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
+#elif defined(RDNA3_5)
+ return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
+#elif defined(RDNA3)
+ return ggml_cuda_mmq_get_config_rdna3(type, J, fallback);
+#else
+ return ggml_cuda_mmq_get_config_rdna2(type, J, fallback);
+#endif // CDNA
+#else
+#ifdef BLACKWELL_MMA_AVAILABLE
+ // only src1 at Q4 uses the native FP4 config, higher precisions keep src1 at Q8_1
+ if (prec_src1 != GGML_PREC_Q4 && (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4)) {
+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
+ }
+ return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);
+#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
+#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A
+ return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback);
+#else
+ return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback);
+#endif // BLACKWELL_MMA_AVAILABLE
+#endif // GGML_USE_HIP
+ GGML_UNUSED_VARS(type, J, fallback, prec_src1);
+}
+
+static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).type;
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type;
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).nthreads;
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).occupancy;
+}
+
+static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).I;
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).I;
+}
+
+static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).J;
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).J;
+}
+
+static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).sram_layout;
+}
+
+static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).sram_layout;
+}
+
+static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).K_vram;
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).K_vram;
+}
+
+static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).stream_k;
+}
+
+static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).stream_k;
+}
+
+static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).fallback;
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).fallback;
+}
+
+// ---------------------------------------------------------------------------------------------
+
+static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc, prec_src1));
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, prec_src1));
+}
+
+static __host__ bool ggml_cuda_mmq_needs_fallback(const int64_t nrows_x) {
+ return nrows_x % 128 != 0;
+}
+
+static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {
+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).rows_per_warp();
+}
+
+#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_0 + I/QI4_0, 0}
+#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_1 + I/QI4_1, 0}
+#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_0 + I/(QI8_0/2), 0}
+#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*4/QI8_0 + I/(QI8_0/4), 0}
+#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_1 + I/(QI8_1/2), 0}
+#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K + I, 0}
+#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I, I*MMQ_TILE_NE_K/8 + I/8}
+#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_K, I*MMQ_TILE_NE_K/8 + I/8}
+#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI5_K + I/QI5_K, I*MMQ_TILE_NE_K/8 + I/8}
+#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI6_K + I/QI6_K, I*MMQ_TILE_NE_K/8 + I/8}
+
+static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) {
+ switch (type) {
+ case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0;
+ case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1;
+ case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1;
+ case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1;
+ case GGML_TYPE_NVFP4: return MMQ_DP4A_TXS_Q8_0_16;
+ case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K;
+ case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K;
+ case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K;
+ case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K;
+ case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K;
+ case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;
+ case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16;
+ case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;
+ case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;
+ default: return tile_x_sizes{0, 0, 0};
+ }
+}
+
+// FIXME temporary until all combinations of data types and GPUs can use the MMA data layout
+static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config & config, const int cc) {
+ if (config.use_mma_data_layout(cc)) {
+ return config.I * ggml_cuda_mmq_get_sram_stride(config.sram_layout) * 4;
+ }
+ const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(config.type, config.I);
+ return (txs.qs + txs.dm + txs.sc) * 4;
+}
+
+// ------------------------------------------------------------
+
+#include "mmq-load-tiles.cuh"
+#include "mmq-vec-dot.cuh"
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(
+ const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, dst_t * __restrict__ dst,
+ const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
+
+ const bool y_scale_used = y_scale != nullptr;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (j > j_max) {
+ return;
+ }
+
+#pragma unroll
+ for (int i0 = 0; i0 < I; i0 += warp_size) {
+ const int i = i0 + threadIdx.x;
+
+ if (fallback && i > i_max) {
+ continue;
+ }
+
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (y_scale_used) {
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
+ } else {
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
+ }
+ } else {
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);
+ GGML_UNUSED(y_scale_used);
+ }
+ }
+ }
+}
+
+template<ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float>
+static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
+ const float * __restrict__ sum, const int * __restrict__ ids_dst, dst_t * __restrict__ dst,
+ const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
+
+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;
+#else
+ typedef tile<16, 8, int> tile_C;
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
+
+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1);
+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+
+ const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I);
+
+ const bool y_scale_used = y_scale != nullptr;
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+#pragma unroll
+ for (int n = 0; n < ntx; ++n) {
+#pragma unroll
+ for (int l = 0; l < tile_C::ne; ++l) {
+ const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l);
+
+ if (j > j_max) {
+ continue;
+ }
+
+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);
+
+ if (fallback && i > i_max) {
+ continue;
+ }
+
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (y_scale_used) {
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]);
+ } else {
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);
+ }
+ } else {
+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);
+ GGML_UNUSED(y_scale_used);
+ }
+ }
+ }
+ }
+}
+
+// -------------------------------------------------------------------------------------------------------------------------------------
+
+// TODO remove this struct and use ggml_cuda_mmq_sram_layout instead.
+struct ggml_cuda_mmq_util_funcs {
+ int vdr;
+ ggml_cuda_mmq_load_tiles_t load_tiles;
+ ggml_cuda_mmq_vec_dot_t vec_dot;
+ ggml_cuda_mmq_write_back_t write_back;
+
+ constexpr __host__ __device__ ggml_cuda_mmq_util_funcs(
+ int vdr, ggml_cuda_mmq_load_tiles_t load_tiles, ggml_cuda_mmq_vec_dot_t vec_dot, ggml_cuda_mmq_write_back_t write_back) :
+ vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {}
+};
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() {
+ if (!ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) {
+ switch (type) {
+ case GGML_TYPE_Q1_0:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q1_0_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q2_0:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q2_0_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q4_0:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q4_0_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q4_1:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q4_1_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q5_0:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q5_0_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q5_1:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q5_1_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q8_0:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q8_0_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+// ---------------------------------------------------------------------------------------------
+ case GGML_TYPE_Q2_K:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q2_K_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q3_K:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q3_K_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q4_K:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q4_K_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q5_K:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q5_K_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q6_K:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_Q6_K_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+// ---------------------------------------------------------------------------------------------
+ case GGML_TYPE_IQ1_S:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ1_S_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ2_XXS:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ2_XXS_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ2_XS:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ2_XS_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ2_S:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ2_S_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ3_XXS:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ3_XXS_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ3_S:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ3_S_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ4_XS:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ4_XS_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ4_NL:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_IQ4_NL_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+// ---------------------------------------------------------------------------------------------
+ case GGML_TYPE_MXFP4:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_MXFP4_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ case GGML_TYPE_NVFP4:
+ return ggml_cuda_mmq_util_funcs(
+ VDR_NVFP4_Q8_1_MMQ,
+ ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback, prec_src1>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
+ default:
+ return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);
+ }
+ }
+
+// ---------------------------------------------------------------------------------------------
+
+#ifdef BLACKWELL_MMA_AVAILABLE
+ switch (type) {
+ case GGML_TYPE_MXFP4:
+ if (prec_src1 == GGML_PREC_Q4) {
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_mxfp4_fp4<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ }
+ break;
+ case GGML_TYPE_NVFP4:
+ if (prec_src1 == GGML_PREC_Q4) {
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_nvfp4_nvfp4<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ }
+ break;
+ default:
+ break;
+ }
+#endif // BLACKWELL_MMA_AVAILABLE
+
+// ---------------------------------------------------------------------------------------------
+
+ switch (type) {
+ case GGML_TYPE_Q1_0:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q2_0:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q4_0:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_DS4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q4_1:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q5_0:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q5_1:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q8_0:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+// ---------------------------------------------------------------------------------------------
+ case GGML_TYPE_Q2_K:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q3_K:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q4_K:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q5_K:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_Q6_K:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+// ---------------------------------------------------------------------------------------------
+ case GGML_TYPE_IQ1_S:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ2_XXS:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ2_XS:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ2_S:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ3_XXS:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ3_S:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ4_XS:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_IQ4_NL:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+// ---------------------------------------------------------------------------------------------
+ case GGML_TYPE_MXFP4:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,
+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ case GGML_TYPE_NVFP4:
+ return ggml_cuda_mmq_util_funcs(
+ -1,
+ ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback, prec_src1>,
+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
+ default:
+ return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);
+ }
+}
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+static constexpr __device__ int ggml_cuda_mmq_get_vdr() {
+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().vdr;
+}
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() {
+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().load_tiles;
+}
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() {
+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().vec_dot;
+}
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() {
+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().write_back;
+}
+
+// ---------------------------------------------------------------------------------------------
+
+template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1, typename dst_t>
+static __device__ __forceinline__ void mul_mat_q_process_tile(
+ const char * __restrict__ x, const int offset_x, const int * __restrict__ y,
+ const int * __restrict__ ids_dst, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,
+ const float * __restrict__ y_scale,
+ const int stride_row_x, const int ncols_y, const int stride_col_dst,
+ const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {
+
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
+ constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles<type, J, fallback, prec_src1>();
+ constexpr ggml_cuda_mmq_vec_dot_t vec_dot = ggml_cuda_mmq_get_vec_dot<type, J, fallback, prec_src1>();
+ constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back<type, J, fallback, prec_src1>();
+
+ extern __shared__ int data_mul_mat_q[];
+ int * tile_y = data_mul_mat_q + J;
+ int * tile_x = tile_y + GGML_PAD(J*MMQ_TILE_Y_K, nwarps*warp_size);
+
+#if defined(BLACKWELL_MMA_AVAILABLE)
+ // FP4 tile stores 8 blocks. src1 above Q4 uses the generic
+ // Q8_1 tile layout instead of the packed FP4 tile.
+ constexpr int ne_block = ((type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4) && prec_src1 == GGML_PREC_Q4) ?
+ QK_FP4_MMQ : QK8_1_MMQ;
+#else
+ constexpr int ne_block = QK8_1_MMQ;
+#endif // defined(BLACKWELL_MMA_AVAILABLE)
+
+ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1);
+ constexpr int blocks_per_iter = ITER_K / qk;
+
+ float sum[J*I / (nwarps*warp_size)] = {0.0f};
+
+ constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int);
+
+ for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) {
+ load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x);
+ {
+ const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz;
+#pragma unroll
+ for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {
+ int l = l0 + threadIdx.y*warp_size + threadIdx.x;
+
+ tile_y[l] = by0[l];
+ }
+ }
+
+ __syncthreads();
+
+ vec_dot(tile_x, tile_y, sum, 0);
+
+ __syncthreads();
+
+ {
+ const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + sz);
+#pragma unroll
+ for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {
+ int l = l0 + threadIdx.y*warp_size + threadIdx.x;
+
+ tile_y[l] = by0[l];
+ }
+ }
+
+ __syncthreads();
+
+ vec_dot(tile_x, tile_y, sum, MMQ_TILE_NE_K);
+
+ __syncthreads();
+ }
+
+ if (fixup) {
+ write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);
+ } else if constexpr (std::is_same_v<dst_t, float>) {
+ write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
+ } else if constexpr (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) {
+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
+ } else {
+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
+ }
+}
+
+
+// A flat MoE grid runs only the (expert, column tile) pairs listed in tile_map, see ggml_cuda_launch_mm_ids_tile_map.
+static __device__ __forceinline__ void mmq_flat_tile(const int2 * __restrict__ tile_map, int & zt, int & jt) {
+ if (tile_map) {
+ const int2 t = tile_map[jt];
+ zt = t.x;
+ jt = t.y;
+ }
+}
+
+// The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t>
+__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1))
+static __global__ void mul_mat_q(
+ const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,
+ const int32_t * __restrict__ expert_bounds, const int2 * __restrict__ tile_map, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,
+ const float * __restrict__ y_scale,
+ const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,
+ const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
+ const uint3 sample_ratio, const uint3 nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
+ const uint3 ntx) {
+
+ // Skip unused template specializations for faster compilation:
+ if (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type == GGML_TYPE_COUNT) {
+ NO_DEVICE_CODE;
+ return;
+ }
+
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
+
+ const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y
+
+ // Initialize the ids for writing back data with just the index.
+ // For regular matrix multiplications this is never changed.
+ // For MoE the correct indices are loaded from ids_dst.
+ extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory.
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
+ const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
+
+ if (j0 + nwarps*warp_size > J && j >= J) {
+ break;
+ }
+
+ ids_dst_shared[j] = j;
+ }
+ __syncthreads();
+
+ if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {
+ const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
+ const int wt = tmp2.x;
+ int zt = tmp2.y;
+ int jt = blockIdx.y;
+ const int it = blockIdx.x;
+ mmq_flat_tile(tile_map, zt, jt);
+
+ // Defaults for regular matrix multiplication:
+ int col_low = 0;
+ int col_high = ncols_dst;
+ int col_diff = ncols_dst;
+ int offset_y = wt*stride_sample_y + zt*stride_channel_y;
+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
+ int offset_y_scale;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
+ } else {
+ GGML_UNUSED(offset_y_scale);
+ }
+
+ if (ids_dst) {
+ col_low = expert_bounds[zt + 0];
+ col_high = expert_bounds[zt + 1];
+ col_diff = col_high - col_low;
+
+ offset_y = 0;
+ offset_dst = 0;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale = 0;
+ }
+
+ if (jt*J >= col_diff) {
+ return;
+ }
+
+ // __syncthreads(); // There is no previous tile that could cause a race condition.
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
+ const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
+
+ if (j0 + nwarps*warp_size > J && j >= J) {
+ break;
+ }
+
+ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];
+ }
+ __syncthreads();
+ }
+
+ offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int));
+ offset_dst += it*I;
+ const float * y_scale_tile = nullptr;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale += col_low + jt*J;
+ y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;
+ }
+
+ const int tile_x_max_i = nrows_x - it*I - 1;
+ const int tile_y_max_j = col_diff - jt*J - 1;
+
+ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
+
+ constexpr bool fixup = false;
+ mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>
+ (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
+ stride_row_x, ncols_y, stride_col_dst,
+ tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z);
+ return;
+ }
+
+ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1);
+ constexpr int blocks_per_iter = ITER_K / qk;
+
+ // kbc == k block continuous, current index in continuous ijk space.
+ int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
+ int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
+
+ kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter;
+ kbc_stop -= fastmodulo(kbc_stop, blocks_per_ne00) % blocks_per_iter;
+
+ // kb0 == k index when doing the matrix multiplication for an output tile.
+ int kb0_start = fastmodulo(kbc, blocks_per_ne00);
+ int kb0_stop = min(blocks_per_ne00.z, uint32_t(kb0_start + kbc_stop - kbc));
+ while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) {
+ int tmp = fastdiv(kbc, blocks_per_ne00);
+ uint2 tmp2 = fast_div_modulo(tmp, ntx);
+ int jt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nchannels_y);
+ int zt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nsamples_y);
+ const int wt = tmp2.y;
+ const int it = tmp2.x;
+ mmq_flat_tile(tile_map, zt, jt);
+
+ // Defaults for regular matrix multiplication:
+ int col_low = 0;
+ int col_high = ncols_dst;
+ int col_diff = ncols_dst;
+ int offset_y = wt*stride_sample_y + zt*stride_channel_y;
+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
+ int offset_y_scale;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
+ } else {
+ GGML_UNUSED(offset_y_scale);
+ }
+
+ if (ids_dst) {
+ col_low = expert_bounds[zt + 0];
+ col_high = expert_bounds[zt + 1];
+ col_diff = col_high - col_low;
+
+ offset_y = 0;
+ offset_dst = 0;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale = 0;
+ }
+
+ if (jt*J >= col_diff) {
+ kbc += blocks_per_ne00.z;
+ kbc -= fastmodulo(kbc, blocks_per_ne00);
+
+ kb0_start = 0;
+ kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc));
+
+ continue;
+ }
+
+ __syncthreads();
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
+ const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
+
+ if (j0 + nwarps*warp_size > J && j >= J) {
+ break;
+ }
+
+ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];
+ }
+ __syncthreads();
+ }
+
+ offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));
+ offset_dst += it*I;
+ const float * y_scale_tile = nullptr;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale += col_low + jt * J;
+ y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;
+ }
+
+ const int tile_x_max_i = nrows_x - it*I - 1;
+ const int tile_y_max_j = col_diff - jt*J - 1;
+
+ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
+
+ constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
+ mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>
+ (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
+ stride_row_x, ncols_y, stride_col_dst,
+ tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);
+
+ kbc += blocks_per_ne00.z;
+ kbc -= fastmodulo(kbc, blocks_per_ne00);
+
+ kb0_start = 0;
+ kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc));
+ }
+
+ if (kbc >= kbc_stop) {
+ return;
+ }
+
+ int tmp = fastdiv(kbc, blocks_per_ne00);
+ uint2 tmp2 = fast_div_modulo(tmp, ntx);
+ int jt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nchannels_y);
+ int zt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nsamples_y);
+ const int wt = tmp2.y;
+ const int it = tmp2.x;
+ mmq_flat_tile(tile_map, zt, jt);
+
+ // Defaults for regular matrix multiplication:
+ int col_low = 0;
+ int col_high = ncols_dst;
+ int col_diff = ncols_dst;
+ int offset_y = wt*stride_sample_y + zt*stride_channel_y;
+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
+ int offset_y_scale;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
+ } else {
+ GGML_UNUSED(offset_y_scale);
+ }
+
+ if (ids_dst) {
+ col_low = expert_bounds[zt + 0];
+ col_high = expert_bounds[zt + 1];
+ col_diff = col_high - col_low;
+
+ offset_y = 0;
+ offset_dst = 0;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale = 0;
+ }
+
+ if (jt*J >= col_diff) {
+ return;
+ }
+
+ // The memory layout for the fixup buffer is always contiguous, therefore reset ids:
+ __syncthreads();
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
+ const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
+
+ if (j0 + nwarps*warp_size > J && j >= J) {
+ break;
+ }
+
+ ids_dst_shared[j] = j;
+ }
+ __syncthreads();
+ }
+
+ offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));
+ offset_dst += it*I;
+ const float * y_scale_tile = nullptr;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ offset_y_scale += col_low + jt * J;
+ y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;
+ }
+
+ const int tile_x_max_i = nrows_x - it*I - 1;
+ const int tile_y_max_j = col_diff - jt*J - 1;
+
+ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
+
+ constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
+ mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>
+ (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
+ stride_row_x, ncols_y, stride_col_dst,
+ tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);
+}
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1)/2, 1)
+static __global__ void mul_mat_q_stream_k_fixup(
+ const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst,
+ float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst,
+ const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y,
+ const int stride_sample_dst, const uint3 ntx) {
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+ constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / 2) / warp_size;
+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1);
+ constexpr int blocks_per_iter = ITER_K / qk;
+
+ float sum[J / nwarps] = {0.0f};
+ const int i = blockIdx.y*warp_size + threadIdx.x;
+
+ const int nty = (nrows_x + I - 1) / I;
+
+ const int bidx0 = blockIdx.x;
+
+ // kbc == k block continuous, current index in continuous ijk space.
+ int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
+ int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
+
+ kbc0 -= fastmodulo(kbc0, blocks_per_ne00) % blocks_per_iter;
+ kbc0_stop -= fastmodulo(kbc0_stop, blocks_per_ne00) % blocks_per_iter;
+
+ const bool did_not_have_any_data = kbc0 == kbc0_stop;
+ const bool wrote_beginning_of_tile = fastmodulo(kbc0, blocks_per_ne00) == 0;
+ const bool did_not_write_last = fastdiv(kbc0, blocks_per_ne00) == fastdiv(kbc0_stop, blocks_per_ne00) && fastmodulo(kbc0_stop, blocks_per_ne00) != 0;
+ if (did_not_have_any_data || wrote_beginning_of_tile || did_not_write_last) {
+ return;
+ }
+
+ bool any_fixup = false;
+
+ // Iterate over previous blocks and sum up partial sums written to fixup buffer.
+ // All CUDA blocks that get here must have a previous block that needs a fixup.
+ int bidx = bidx0 - 1;
+ int kbc_stop = kbc0;
+ while(true) {
+ int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
+ kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter;
+
+ if (kbc == kbc_stop) { // Did not have any data.
+ bidx--;
+ kbc_stop = kbc;
+ continue;
+ }
+
+ any_fixup = true;
+
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ sum[j0/nwarps] += tmp_last_tile[bidx*(J*I) + j*I + i];
+ }
+
+ // If this block started in a previous tile we are done and don't need to combine additional partial results.
+ if (fastmodulo(kbc, blocks_per_ne00) == 0 || fastdiv(kbc, blocks_per_ne00) < fastdiv(kbc0, blocks_per_ne00)) {
+ break;
+ }
+ bidx--;
+ kbc_stop = kbc;
+ }
+
+ if (!any_fixup) {
+ return;
+ }
+
+ int tmp = fastdiv(kbc0, blocks_per_ne00);
+ uint2 tmp2 = fast_div_modulo(tmp, ntx);
+ const int jt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nchannels_y);
+ const int zt = tmp2.y;
+ tmp = tmp2.x;
+ tmp2 = fast_div_modulo(tmp, nsamples_y);
+ const int wt = tmp2.y;
+ const int it = tmp2.x;
+
+ if (!ids_dst) {
+ const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;
+ dst += offset_dst;
+
+ const int i_max = nrows_x - it*I - 1;
+ const int j_max = ncols_dst - jt*J - 1;
+ if (fallback && i > i_max) {
+ return;
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (j > j_max) {
+ return;
+ }
+
+ dst[j*stride_col_dst + i] += sum[j0/nwarps];
+ }
+ return;
+ }
+
+ __shared__ int ids_dst_shared[J];
+ const int col_low = expert_bounds[zt + 0];
+ const int col_high = expert_bounds[zt + 1];
+ const int col_diff = col_high - col_low;
+
+ for (int j = threadIdx.y*warp_size + threadIdx.x; j < J; j += nwarps*warp_size) {
+ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];
+ }
+ __syncthreads();
+
+ const int offset_dst = it*I;
+ dst += offset_dst;
+
+ const int i_max = nrows_x - it*I - 1;
+ const int j_max = col_diff - jt*J - 1;
+ if (fallback && i > i_max) {
+ return;
+ }
+
+#pragma unroll
+ for (int j0 = 0; j0 < J; j0 += nwarps) {
+ const int j = j0 + threadIdx.y;
+
+ if (j > j_max) {
+ return;
+ }
+
+ dst[ids_dst_shared[j]*stride_col_dst + i] += sum[j0/nwarps];
+ }
+}
+
+// FreeToken: copies an fp32 result with dst's layout into dst
+template <typename dst_t>
+static __global__ void mmq_store_dst(const float * __restrict__ src, dst_t * __restrict__ dst, const int nrows, const int stride) {
+ const int64_t col = blockIdx.x;
+ const int row = blockIdx.y*blockDim.x + threadIdx.x;
+ if (row < nrows) {
+ dst[col*stride + row] = mmq_cast<dst_t>(src[col*stride + row]);
+ }
+}
+
+struct mmq_args {
+ const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; mmq_dst dst;
+ const float * y_scale;
+ int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst;
+ int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst;
+ int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;
+ int64_t ncols_max;
+ int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance.
+ const int2 * tile_map = nullptr; // Flat MoE grid: the (expert, column tile) of each of the ntx_flat grid columns.
+ int ntx_flat = 0;
+};
+
+static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {
+ const size_t nbs_ids = config.J*sizeof(int);
+ const size_t nbs_x = ggml_cuda_mmq_get_nbytes_shared_x(config, cc);
+ const size_t nbs_y = config.J * (sizeof(block_q8_1_mmq));
+ return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int));
+}
+
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
+static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+ const int id = ggml_cuda_get_device();
+ const int cc = ggml_cuda_info().devices[id].cc;
+ const int nsm = ggml_cuda_info().devices[id].nsm;
+ const int warp_size = ggml_cuda_info().devices[id].warp_size;
+
+ const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1);
+ GGML_ASSERT(config.nthreads % warp_size == 0);
+ const int nwarps = config.nthreads / warp_size;
+ const int nbytes_shared = mmq_get_nbytes_shared(config, cc);
+
+ const dim3 block_dims(warp_size, nwarps, 1);
+
+ // a flat MoE grid puts all experts into one channel of ntx_flat column tiles
+ const int64_t nchannels_grid = args.tile_map ? 1 : args.nchannels_y;
+ const int nty = (args.nrows_x + config.I - 1) / config.I;
+ const int ntx = args.tile_map ? args.ntx_flat : (args.ncols_max + config.J - 1) / config.J;
+ const int ntzw = nchannels_grid * args.nsamples_y;
+ const dim3 block_nums_xy_tiling(nty, ntx, ntzw);
+
+ GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);
+ GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0);
+ const int channel_ratio = args.nchannels_y / args.nchannels_x;
+ const int sample_ratio = args.nsamples_y / args.nsamples_x;
+
+ const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);
+ const uint3 ntx_fd = init_fastdiv_values(ntx);
+ const uint3 nchannels_y_fd = init_fastdiv_values(nchannels_grid);
+ const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y);
+ const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio);
+ const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);
+
+ const auto run = [&](auto * dst, const dim3 & block_nums, float * tmp_fixup) {
+ using dst_t = std::remove_pointer_t<decltype(dst)>;
+ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1, dst_t>), nbytes_shared);
+ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1, dst_t>), nbytes_shared);
+ mul_mat_q<type, J, fallback, prec_src1, dst_t><<<block_nums, block_dims, nbytes_shared, stream>>>
+ (args.x, args.y, args.ids_dst, args.expert_bounds, args.tile_map, dst, tmp_fixup, args.y_scale,
+ blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
+ channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
+ sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
+ ntx_fd);
+ };
+
+ // FreeToken: the fixup kernel adds partial sums into an fp32 dst, so a bf16 dst that needs it is computed in fp32 first.
+ // An fp16 dst always is, unless GGML_MMQ_F16_DST builds the fp16 kernels.
+ const auto launch = [&](const dim3 & block_nums, const bool fixup, float * tmp_fixup) {
+ bool via_f32 = fixup && args.dst.type != GGML_TYPE_F32;
+#ifndef GGML_MMQ_F16_DST
+ via_f32 = via_f32 || args.dst.type == GGML_TYPE_F16;
+#endif // GGML_MMQ_F16_DST
+ ggml_cuda_pool_alloc<float> dst_f32(ctx.pool(id));
+ float * dst_float = args.dst.type == GGML_TYPE_F32 ? (float *) args.dst.p : nullptr;
+ if (via_f32) {
+ GGML_ASSERT(args.ids_dst || (args.nchannels_y == 1 && args.nsamples_y == 1));
+ dst_float = dst_f32.alloc(args.ncols_dst * args.nrows_dst);
+ }
+
+ if (dst_float) {
+ run(dst_float, block_nums, tmp_fixup);
+ } else if (args.dst.type == GGML_TYPE_BF16) {
+ run((nv_bfloat16 *) args.dst.p, block_nums, tmp_fixup);
+ }
+#ifdef GGML_MMQ_F16_DST
+ else {
+ run((half *) args.dst.p, block_nums, tmp_fixup);
+ }
+#endif // GGML_MMQ_F16_DST
+
+ if (fixup) {
+ CUDA_CHECK(cudaGetLastError());
+ const dim3 block_nums_fixup(block_nums.x, config.I/warp_size, 1);
+ const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
+ mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
+ (args.ids_dst, args.expert_bounds, dst_float, tmp_fixup, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
+ args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,
+ ntx_fd);
+ }
+
+ if (via_f32) {
+ CUDA_CHECK(cudaGetLastError());
+ const dim3 block_nums_store(args.ncols_dst, (args.nrows_x + 255) / 256, 1);
+ if (args.dst.type == GGML_TYPE_BF16) {
+ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (nv_bfloat16 *) args.dst.p, args.nrows_x, args.nrows_dst);
+ } else {
+ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (half *) args.dst.p, args.nrows_x, args.nrows_dst);
+ }
+ }
+ };
+
+ if (!config.stream_k) {
+ launch(block_nums_xy_tiling, false, nullptr);
+ return;
+ }
+
+ // For the stream-k kernel it is possible to run it with tiling by setting the number of CUDA blocks equal to the number of tiles.
+ // This is worthwhile if the efficiency of tiling is high and skipping the fixup kernel is more important.
+ const int ntiles_dst = ntx * nty * ntzw;
+ const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;
+ const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);
+ // The fixup kernel does not read tile_map, so a flat MoE grid always runs one tile per block.
+ const dim3 block_nums_stream_k(args.tile_map || (GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90) ? ntiles_dst : nsm, 1, 1);
+
+ GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.
+
+ const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0;
+
+ ggml_cuda_pool & pool = ctx.pool(id);
+ ggml_cuda_pool_alloc<float> tmp_fixup(pool);
+ if (fixup_needed) {
+ tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);
+ }
+
+ launch(block_nums_stream_k, fixup_needed, tmp_fixup.ptr);
+}
+
+template <ggml_type type, bool fallback, ggml_prec prec_src1>
+void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+ switch (args.J_best) {
+ case 8:
+ launch_mul_mat_q<type, 8, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 16:
+ launch_mul_mat_q<type, 16, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 24:
+ launch_mul_mat_q<type, 24, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 32:
+ launch_mul_mat_q<type, 32, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 40:
+ launch_mul_mat_q<type, 40, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 48:
+ launch_mul_mat_q<type, 48, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 56:
+ launch_mul_mat_q<type, 56, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 64:
+ launch_mul_mat_q<type, 64, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 72:
+ launch_mul_mat_q<type, 72, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 80:
+ launch_mul_mat_q<type, 80, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 88:
+ launch_mul_mat_q<type, 88, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 96:
+ launch_mul_mat_q<type, 96, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 104:
+ launch_mul_mat_q<type, 104, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 112:
+ launch_mul_mat_q<type, 112, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 120:
+ launch_mul_mat_q<type, 120, fallback, prec_src1>(ctx, args, stream);
+ break;
+ case 128:
+ launch_mul_mat_q<type, 128, fallback, prec_src1>(ctx, args, stream);
+ break;
+ default:
+ fprintf(stderr, "J_best=%d\n", args.J_best);
+ GGML_ABORT("fatal error");
+ }
+}
+
+template <ggml_type type, ggml_prec prec_src1>
+void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+ if (ggml_cuda_mmq_needs_fallback(args.nrows_x)) {
+ constexpr bool fallback = true;
+ mul_mat_q_switch_J<type, fallback, prec_src1>(ctx, args, stream);
+ } else {
+ constexpr bool fallback = false;
+ mul_mat_q_switch_J<type, fallback, prec_src1>(ctx, args, stream);
+ }
+}
+
+#define DECL_MMQ_CASE(type) \
+ template void mul_mat_q_case<type, GGML_PREC_Q8>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \
+
+// FP4 variant: uses native FP4 MMA instead of keeping src1 at Q8_1.
+#define DECL_MMQ_CASE_W4A4(type) \
+ template void mul_mat_q_case<type, GGML_PREC_Q4>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \
+
+extern DECL_MMQ_CASE(GGML_TYPE_Q1_0);
+extern DECL_MMQ_CASE(GGML_TYPE_Q2_0);
+extern DECL_MMQ_CASE(GGML_TYPE_Q4_0);
+extern DECL_MMQ_CASE(GGML_TYPE_Q4_1);
+extern DECL_MMQ_CASE(GGML_TYPE_Q5_0);
+extern DECL_MMQ_CASE(GGML_TYPE_Q5_1);
+extern DECL_MMQ_CASE(GGML_TYPE_Q8_0);
+// -----------------------------------------
+extern DECL_MMQ_CASE(GGML_TYPE_Q2_K);
+extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);
+extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);
+extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);
+extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
+// -----------------------------------------
+extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
+extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);
+// -----------------------------------------
+extern DECL_MMQ_CASE(GGML_TYPE_MXFP4);
+extern DECL_MMQ_CASE(GGML_TYPE_NVFP4);
+extern DECL_MMQ_CASE_W4A4(GGML_TYPE_MXFP4);
+extern DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4);
+
+// -------------------------------------------------------------------------------------------------------------------------
+
+// FreeToken: under ids, ncols_max > 0 bounds the tokens any one expert gets and sizes the tile grid; otherwise the grid is flat, see ggml_cuda_launch_mm_ids_tile_map
+void ggml_cuda_mul_mat_q(
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
+ int64_t ncols_max = -1);
+
+bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cu
new file mode 100644
index 00000000..f3a3ccc6
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cu
@@ -0,0 +1,1626 @@
+#include "mmvq.cuh"
+#include "convert.cuh"
+#include "quantize.cuh"
+#include "unary.cuh"
+#include "vecdotq.cuh"
+
+#include <cstdint>
+#include <type_traits>
+
+// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel
+// has little exposed latency left to hide and the extra requests cost more than they save.
+// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031
+#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
+// returns true only for those quants that benefit from prefetch and false otherwise
+static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_MXFP4:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static __device__ __forceinline__ void mmvq_prefetch_l2(const void * p) {
+ asm volatile("prefetch.global.L2 [%0];" :: "l"(p));
+}
+#endif
+
+typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs);
+
+static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_Q1_0: return vec_dot_q1_0_q8_1;
+ case GGML_TYPE_Q2_0: return vec_dot_q2_0_q8_1;
+ case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1;
+ case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1;
+ case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1;
+ case GGML_TYPE_Q5_1: return vec_dot_q5_1_q8_1;
+ case GGML_TYPE_Q8_0: return vec_dot_q8_0_q8_1;
+ case GGML_TYPE_MXFP4: return vec_dot_mxfp4_q8_1;
+ case GGML_TYPE_NVFP4: return vec_dot_nvfp4_q8_1;
+ case GGML_TYPE_Q2_K: return vec_dot_q2_K_q8_1;
+ case GGML_TYPE_Q3_K: return vec_dot_q3_K_q8_1;
+ case GGML_TYPE_Q4_K: return vec_dot_q4_K_q8_1;
+ case GGML_TYPE_Q5_K: return vec_dot_q5_K_q8_1;
+ case GGML_TYPE_Q6_K: return vec_dot_q6_K_q8_1;
+ case GGML_TYPE_IQ2_XXS: return vec_dot_iq2_xxs_q8_1;
+ case GGML_TYPE_IQ2_XS: return vec_dot_iq2_xs_q8_1;
+ case GGML_TYPE_IQ2_S: return vec_dot_iq2_s_q8_1;
+ case GGML_TYPE_IQ3_XXS: return vec_dot_iq3_xxs_q8_1;
+ case GGML_TYPE_IQ1_S: return vec_dot_iq1_s_q8_1;
+ case GGML_TYPE_IQ1_M: return vec_dot_iq1_m_q8_1;
+ case GGML_TYPE_IQ4_NL: return vec_dot_iq4_nl_q8_1;
+ case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1;
+ case GGML_TYPE_IQ3_S: return vec_dot_iq3_s_q8_1;
+ default: return nullptr;
+ }
+}
+
+static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_Q1_0: return VDR_Q1_0_Q8_1_MMVQ;
+ case GGML_TYPE_Q2_0: return VDR_Q2_0_Q8_1_MMVQ;
+ case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ;
+ case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ;
+ case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ;
+ case GGML_TYPE_Q5_1: return VDR_Q5_1_Q8_1_MMVQ;
+ case GGML_TYPE_Q8_0: return VDR_Q8_0_Q8_1_MMVQ;
+ case GGML_TYPE_MXFP4: return VDR_MXFP4_Q8_1_MMVQ;
+ case GGML_TYPE_NVFP4: return VDR_NVFP4_Q8_1_MMVQ;
+ case GGML_TYPE_Q2_K: return VDR_Q2_K_Q8_1_MMVQ;
+ case GGML_TYPE_Q3_K: return VDR_Q3_K_Q8_1_MMVQ;
+ case GGML_TYPE_Q4_K: return VDR_Q4_K_Q8_1_MMVQ;
+ case GGML_TYPE_Q5_K: return VDR_Q5_K_Q8_1_MMVQ;
+ case GGML_TYPE_Q6_K: return VDR_Q6_K_Q8_1_MMVQ;
+ case GGML_TYPE_IQ2_XXS: return VDR_IQ2_XXS_Q8_1_MMVQ;
+ case GGML_TYPE_IQ2_XS: return VDR_IQ2_XS_Q8_1_MMVQ;
+ case GGML_TYPE_IQ2_S: return VDR_IQ2_S_Q8_1_MMVQ;
+ case GGML_TYPE_IQ3_XXS: return VDR_IQ3_XXS_Q8_1_MMVQ;
+ case GGML_TYPE_IQ3_S: return VDR_IQ3_S_Q8_1_MMVQ;
+ case GGML_TYPE_IQ4_NL: return VDR_IQ4_NL_Q8_1_MMVQ;
+ case GGML_TYPE_IQ4_XS: return VDR_IQ4_XS_Q8_1_MMVQ;
+ default: return 1;
+ }
+}
+
+enum mmvq_parameter_table_id {
+ MMVQ_PARAMETERS_GENERIC = 0,
+ MMVQ_PARAMETERS_TURING,
+ MMVQ_PARAMETERS_GCN,
+ MMVQ_PARAMETERS_RDNA2,
+ MMVQ_PARAMETERS_RDNA3_0,
+ MMVQ_PARAMETERS_RDNA4,
+ MMVQ_PARAMETERS_GB10
+};
+
+static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
+#if defined(RDNA4)
+ return MMVQ_PARAMETERS_RDNA4;
+#elif defined(RDNA3_0)
+ return MMVQ_PARAMETERS_RDNA3_0;
+#elif defined(RDNA2) || defined(RDNA3_5)
+ return MMVQ_PARAMETERS_RDNA2;
+#elif defined(GCN) || defined(CDNA)
+ return MMVQ_PARAMETERS_GCN;
+#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE
+ return MMVQ_PARAMETERS_TURING;
+#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
+ return MMVQ_PARAMETERS_GB10;
+#else
+ return MMVQ_PARAMETERS_GENERIC;
+#endif
+}
+
+static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
+ if (GGML_CUDA_CC_IS_RDNA4(cc)) {
+ return MMVQ_PARAMETERS_RDNA4;
+ }
+ if (GGML_CUDA_CC_IS_RDNA3_0(cc)) {
+ return MMVQ_PARAMETERS_RDNA3_0;
+ }
+ if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+ return MMVQ_PARAMETERS_RDNA2;
+ }
+ if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) {
+ return MMVQ_PARAMETERS_GCN;
+ }
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) {
+ return MMVQ_PARAMETERS_TURING;
+ }
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) {
+ return MMVQ_PARAMETERS_GB10;
+ }
+ return MMVQ_PARAMETERS_GENERIC;
+}
+
+// Per-architecture maximum batch size for which MMVQ should be used for MUL_MAT_ID.
+// Returns a value <= MMVQ_MAX_BATCH_SIZE. Default is MMVQ_MAX_BATCH_SIZE.
+// Check https://github.com/ggml-org/llama.cpp/pull/20905#issuecomment-4145835627 for details
+
+static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_pascal_older(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_IQ1_S: return 6;
+ case GGML_TYPE_IQ1_M: return 6;
+ case GGML_TYPE_IQ2_S: return 4;
+ case GGML_TYPE_IQ2_XS: return 5;
+ case GGML_TYPE_IQ2_XXS: return 5;
+ case GGML_TYPE_IQ3_S: return 4;
+ case GGML_TYPE_IQ3_XXS: return 4;
+ case GGML_TYPE_IQ4_NL: return 6;
+ case GGML_TYPE_IQ4_XS: return 5;
+ case GGML_TYPE_MXFP4: return 4;
+ case GGML_TYPE_NVFP4: return 4;
+ case GGML_TYPE_Q2_K: return 4;
+ case GGML_TYPE_Q3_K: return 4;
+ case GGML_TYPE_Q4_0: return 6;
+ case GGML_TYPE_Q4_1: return 6;
+ case GGML_TYPE_Q4_K: return 5;
+ case GGML_TYPE_Q5_0: return 6;
+ case GGML_TYPE_Q5_1: return 6;
+ case GGML_TYPE_Q5_K: return 5;
+ case GGML_TYPE_Q6_K: return 4;
+ case GGML_TYPE_Q8_0: return 4;
+ default: return MMVQ_MAX_BATCH_SIZE;
+ }
+}
+
+static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_turing_plus(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_IQ2_S: return 7;
+ case GGML_TYPE_IQ3_S: return 6;
+ case GGML_TYPE_IQ3_XXS: return 7;
+ case GGML_TYPE_MXFP4: return 7;
+ case GGML_TYPE_NVFP4: return 8;
+ case GGML_TYPE_Q2_K: return 7;
+ case GGML_TYPE_Q3_K: return 5;
+ default: return MMVQ_MAX_BATCH_SIZE;
+ }
+}
+
+static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_gcn(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_IQ1_S: return 5;
+ case GGML_TYPE_IQ1_M: return 5;
+ case GGML_TYPE_IQ2_S: return 4;
+ case GGML_TYPE_IQ2_XS: return 4;
+ case GGML_TYPE_IQ2_XXS: return 4;
+ case GGML_TYPE_IQ3_S: return 4;
+ case GGML_TYPE_IQ3_XXS: return 4;
+ case GGML_TYPE_IQ4_NL: return 6;
+ case GGML_TYPE_IQ4_XS: return 4;
+ case GGML_TYPE_Q2_K: return 4;
+ case GGML_TYPE_Q3_K: return 4;
+ case GGML_TYPE_Q4_0: return 5;
+ case GGML_TYPE_Q4_1: return 5;
+ case GGML_TYPE_Q4_K: return 4;
+ case GGML_TYPE_Q5_K: return 4;
+ case GGML_TYPE_Q6_K: return 4;
+ case GGML_TYPE_Q8_0: return 4;
+ default: return MMVQ_MAX_BATCH_SIZE;
+ }
+}
+
+static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_cdna(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_IQ2_S: return 5;
+ case GGML_TYPE_IQ2_XS: return 5;
+ case GGML_TYPE_IQ2_XXS: return 5;
+ case GGML_TYPE_IQ3_S: return 4;
+ case GGML_TYPE_IQ3_XXS: return 5;
+ default: return MMVQ_MAX_BATCH_SIZE;
+ }
+}
+
+static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna1_rdna2(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_IQ2_S: return 4;
+ case GGML_TYPE_IQ2_XS: return 4;
+ case GGML_TYPE_IQ2_XXS: return 4;
+ case GGML_TYPE_IQ3_S: return 4;
+ case GGML_TYPE_IQ3_XXS: return 4;
+ case GGML_TYPE_Q2_K: return 7;
+ case GGML_TYPE_Q3_K: return 4;
+ case GGML_TYPE_Q4_K: return 5;
+ case GGML_TYPE_Q5_K: return 6;
+ case GGML_TYPE_Q6_K: return 5;
+ default: return MMVQ_MAX_BATCH_SIZE;
+ }
+}
+
+static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna3(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_IQ1_S: return 6;
+ case GGML_TYPE_IQ1_M: return 6;
+ case GGML_TYPE_IQ2_S: return 4;
+ case GGML_TYPE_IQ2_XS: return 4;
+ case GGML_TYPE_IQ2_XXS: return 4;
+ case GGML_TYPE_IQ3_S: return 4;
+ case GGML_TYPE_IQ3_XXS: return 4;
+ case GGML_TYPE_IQ4_NL: return 6;
+ case GGML_TYPE_IQ4_XS: return 6;
+ case GGML_TYPE_Q4_K: return 4;
+ case GGML_TYPE_Q5_K: return 4;
+ case GGML_TYPE_Q6_K: return 4;
+ default: return MMVQ_MAX_BATCH_SIZE;
+ }
+}
+
+static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type type) {
+ switch (type) {
+ case GGML_TYPE_IQ1_S: return 7;
+ case GGML_TYPE_IQ1_M: return 7;
+ case GGML_TYPE_IQ2_S: return 4;
+ case GGML_TYPE_IQ2_XS: return 4;
+ case GGML_TYPE_IQ2_XXS: return 4;
+ case GGML_TYPE_IQ3_S: return 4;
+ case GGML_TYPE_IQ3_XXS: return 4;
+ case GGML_TYPE_IQ4_NL: return 7;
+ case GGML_TYPE_IQ4_XS: return 5;
+ case GGML_TYPE_MXFP4: return 5;
+ case GGML_TYPE_NVFP4: return 5;
+ case GGML_TYPE_Q3_K: return 4;
+ case GGML_TYPE_Q4_0: return 7;
+ case GGML_TYPE_Q4_1: return 7;
+ case GGML_TYPE_Q4_K: return 4;
+ case GGML_TYPE_Q5_0: return 7;
+ case GGML_TYPE_Q5_1: return 7;
+ case GGML_TYPE_Q5_K: return 5;
+ case GGML_TYPE_Q6_K: return 5;
+ case GGML_TYPE_Q8_0: return 7;
+ default: return MMVQ_MAX_BATCH_SIZE;
+ }
+}
+
+// Host function: returns the max batch size for the current arch+type at runtime.
+int get_mmvq_mmid_max_batch(ggml_type type, int cc) {
+ // NVIDIA: P100, Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID.
+ if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
+ if (cc == GGML_CUDA_CC_PASCAL || cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) {
+ return MMVQ_MAX_BATCH_SIZE;
+ }
+ if (cc >= GGML_CUDA_CC_TURING) {
+ return get_mmvq_mmid_max_batch_turing_plus(type);
+ }
+ return get_mmvq_mmid_max_batch_pascal_older(type);
+ }
+
+ // AMD
+ if (GGML_CUDA_CC_IS_AMD(cc)) {
+ if (GGML_CUDA_CC_IS_RDNA4(cc)) {
+ return get_mmvq_mmid_max_batch_rdna4(type);
+ }
+ if (GGML_CUDA_CC_IS_RDNA3(cc)) {
+ return get_mmvq_mmid_max_batch_rdna3(type);
+ }
+ if (GGML_CUDA_CC_IS_RDNA1(cc) || GGML_CUDA_CC_IS_RDNA2(cc)) {
+ return get_mmvq_mmid_max_batch_rdna1_rdna2(type);
+ }
+ if (GGML_CUDA_CC_IS_CDNA(cc)) {
+ return get_mmvq_mmid_max_batch_cdna(type);
+ }
+ if (GGML_CUDA_CC_IS_GCN(cc)) {
+ return get_mmvq_mmid_max_batch_gcn(type);
+ }
+ }
+ return MMVQ_MAX_BATCH_SIZE;
+}
+
+bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
+ if (!ggml_is_quantized(type)) {
+ return false;
+ }
+ // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner.
+ // Only list quant-types MMQ supports, others would fall back to cuBLAS.
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) {
+ switch (type) { // tuned on RTX 4090
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 4;
+ case GGML_TYPE_Q3_K:
+ return ne11 <= 6;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) {
+ switch (type) { // tuned on RTX 5090
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ return ne11 <= 5;
+ case GGML_TYPE_Q5_K:
+ return ne11 <= 6;
+ case GGML_TYPE_Q6_K:
+ return ne11 <= 7;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
+ switch (type) { // tuned on DGX Spark GB10
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 6;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ORIN) {
+ switch (type) { // tuned for Jetson Orin
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ return ne11 <= 1;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_VOLTA) {
+ switch (type) {
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 4;
+ case GGML_TYPE_Q3_K:
+ return ne11 <= 6;
+ case GGML_TYPE_Q4_K:
+ return ne11 <= 5;
+ case GGML_TYPE_Q5_K:
+ return ne11 <= 6;
+ case GGML_TYPE_Q6_K:
+ return ne11 <= 7;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ if (GGML_CUDA_CC_IS_CDNA(cc)) {
+ if (GGML_CUDA_CC_IS_CDNA1(cc)) {
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ return ne11 <= 7;
+ case GGML_TYPE_Q5_1:
+ return ne11 <= 7;
+ case GGML_TYPE_Q8_0:
+ return ne11 <= 6;
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 4;
+ case GGML_TYPE_Q3_K:
+ return ne11 <= 3;
+ case GGML_TYPE_Q4_K:
+ return ne11 <= 2;
+ case GGML_TYPE_Q5_K:
+ return ne11 <= 3;
+ case GGML_TYPE_Q6_K:
+ return ne11 <= 4;
+ case GGML_TYPE_IQ1_S:
+ return ne11 <= 5;
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ4_XS:
+ return ne11 <= 6;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ switch (type) { // tuned for CDNA2
+ case GGML_TYPE_Q2_K:
+ return ne11 <= 5;
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ return ne11 <= 3;
+ case GGML_TYPE_Q6_K:
+ return ne11 <= 5;
+ default:
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+ }
+ }
+ return ne11 <= MMVQ_MAX_BATCH_SIZE;
+}
+
+// Device constexpr: returns the max batch size for the current arch+type at compile time.
+template <ggml_type type>
+static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() {
+#if defined(RDNA4)
+ return get_mmvq_mmid_max_batch_rdna4(type);
+#elif defined(RDNA3)
+ return get_mmvq_mmid_max_batch_rdna3(type);
+#elif defined(RDNA2) || defined(RDNA1)
+ return get_mmvq_mmid_max_batch_rdna1_rdna2(type);
+#elif defined(CDNA)
+ return get_mmvq_mmid_max_batch_cdna(type);
+#elif defined(GCN)
+ return get_mmvq_mmid_max_batch_gcn(type);
+#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_PASCAL || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE)
+ return MMVQ_MAX_BATCH_SIZE;
+#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING
+ return get_mmvq_mmid_max_batch_turing_plus(type);
+#else
+ return get_mmvq_mmid_max_batch_pascal_older(type);
+#endif
+}
+
+static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id, bool small_k = false, bool halve_iters = false) {
+ if (table_id == MMVQ_PARAMETERS_GENERIC) {
+ switch (ncols_dst) {
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ return 4;
+ case 5:
+ case 6:
+ case 7:
+ case 8:
+ return 2;
+ default:
+ return 1;
+ }
+ } else if (table_id == MMVQ_PARAMETERS_GCN) {
+ switch (ncols_dst) {
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ return 2;
+ case 5:
+ case 6:
+ case 7:
+ case 8:
+ default:
+ return 1;
+ }
+ }
+ if (table_id == MMVQ_PARAMETERS_RDNA4) {
+ // nwarps=8 benefits types with simple vec_dot on RDNA4 (ncols_dst=1).
+ // Types with complex vec_dot (Q3_K, IQ2_*, IQ3_*) regress due to register
+ // pressure and lookup table contention at higher thread counts.
+ if (ncols_dst == 1) {
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ return 8;
+ default:
+ return 1;
+ }
+ }
+ return 1;
+ }
+ if (table_id == MMVQ_PARAMETERS_RDNA3_0) {
+ // RDNA3 (W7900): stricter whitelist than RDNA4.
+ // Q2_K / Q5_K / IQ4_XS regress in full quant sweeps.
+ if (ncols_dst == 1) {
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ return 8;
+ case GGML_TYPE_Q6_K:
+ return 2;
+ case GGML_TYPE_IQ4_NL:
+ return 8;
+ default:
+ return 1;
+ }
+ }
+ return 1;
+ }
+ if (table_id == MMVQ_PARAMETERS_TURING) {
+ if (ncols_dst == 1) {
+ switch (type) {
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ return 2;
+ default:
+ return 4;
+ }
+ }
+ switch (ncols_dst) {
+ case 2:
+ case 3:
+ case 4:
+ return 4;
+ case 5:
+ case 6:
+ case 7:
+ case 8:
+ return 2;
+ default:
+ return 1;
+ }
+ }
+ if (table_id == MMVQ_PARAMETERS_GB10) {
+ const int generic = calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC);
+ // Only worth the wider block when it actually retires the K loop in half the trips (Observation)
+ if (ncols_dst == 1 && !small_k && halve_iters) {
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ4_NL:
+ return 2 * generic;
+ default:
+ break;
+ }
+ }
+ return generic;
+ }
+ return 1;
+}
+
+static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) {
+ if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) {
+ switch (ncols_dst) {
+ case 1:
+ return small_k ? nwarps : 1;
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ case 8:
+ return 2;
+ default:
+ return 1;
+ }
+ }
+ return 1;
+}
+
+// FreeToken: dst_t is float, nv_bfloat16 or half
+template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false, typename dst_t = float>
+__launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_q(
+ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst_ptr,
+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,
+ uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
+ const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
+ const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,
+ const uint32_t ids_stride) {
+ const void * GGML_CUDA_RESTRICT vx = vx_ptr;
+ const void * GGML_CUDA_RESTRICT vy = vy_ptr;
+ const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
+ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;
+
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int qi = ggml_cuda_type_traits<type>::qi;
+ constexpr int vdr = get_vdr_mmvq(type);
+ constexpr mmvq_parameter_table_id table_id = get_device_table_id();
+ constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);
+ constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+
+ constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
+
+ const int tid = warp_size*threadIdx.y + threadIdx.x;
+ const int row0 = rows_per_cuda_block*blockIdx.x;
+ const int blocks_per_row_x = ncols_x / qk;
+ constexpr int blocks_per_iter = vdr * nwarps*warp_size / qi;
+
+ const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1;
+ const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y;
+ if (shared_expert) {
+ vx = fusion.shared_up;
+ if constexpr (std::is_same_v<dst_t, float>) {
+ dst = fusion.shared_dst;
+ }
+ stride_col_dst = fusion.shared_stride_col_dst;
+ }
+
+ uint32_t channel_x;
+ uint32_t channel_y;
+ uint32_t sample_dst;
+
+ ggml_cuda_pdl_sync();
+ channel_x = shared_expert ? 0 : ncols_dst == 1 && ids ? ids[channel_dst] : fastdiv(channel_dst, channel_ratio);
+ channel_y = ncols_dst == 1 && ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst;
+ sample_dst = blockIdx.z;
+
+ const uint32_t sample_x = fastdiv(sample_dst, sample_ratio);
+ const uint32_t sample_y = sample_dst;
+
+ bool use_gate = false;
+ bool use_bias = false;
+ bool use_gate_bias = false;
+ bool use_scale = false;
+ bool use_gate_scale = false;
+ [[maybe_unused]] const void * vgate = nullptr;
+ const float * x_bias = nullptr;
+ const float * gate_bias = nullptr;
+ const float * x_scale = nullptr;
+ const float * gate_scale = nullptr;
+ ggml_glu_op active_glu;
+ float glu_limit = 0.0f;
+
+ if constexpr (has_fusion) {
+ use_gate = fusion.gate != nullptr;
+ use_bias = fusion.x_bias != nullptr;
+ use_gate_bias = fusion.gate_bias != nullptr && use_gate;
+ vgate = shared_expert ? fusion.shared_gate : fusion.gate;
+ x_bias = (const float *) fusion.x_bias;
+ gate_bias = (const float *) fusion.gate_bias;
+ active_glu = fusion.glu_op;
+ glu_limit = fusion.glu_limit;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ use_scale = fusion.x_scale != nullptr;
+ use_gate_scale = fusion.gate_scale != nullptr && use_gate;
+ x_scale = (const float *) fusion.x_scale;
+ gate_scale = (const float *) fusion.gate_scale;
+ }
+ }
+
+
+ [[maybe_unused]] float x_biases[ncols_dst] = { 0.0f };
+ [[maybe_unused]] float gate_biases[ncols_dst] = { 0.0f };
+ [[maybe_unused]] float x_scales = 1.0f;
+ [[maybe_unused]] float gate_scales = 1.0f;
+ if constexpr (has_fusion) {
+ // 1. Hide latency by prefetching bias, gates and scales here
+ // 2. load only on threads that won't die after partial sum calculation
+ const uint32_t channel_bias = ids ? channel_x : channel_dst;
+ if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&
+ (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {
+ if (use_bias) {
+ x_bias = x_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;
+#pragma unroll
+ for (int j = 0; j < ncols_dst; ++j) {
+ x_biases[j] = x_bias[j * stride_col_dst + threadIdx.x];
+ }
+ }
+ if (use_gate_bias) {
+ gate_bias = gate_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;
+#pragma unroll
+ for (int j = 0; j < ncols_dst; ++j) {
+ gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x];
+ }
+ }
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (use_scale) {
+ x_scales = x_scale[ids ? channel_x : 0];
+ }
+ if (use_gate_scale) {
+ gate_scales = gate_scale[ids ? channel_x : 0];
+ }
+ }
+ }
+ }
+
+ // partial sum for each thread
+ float tmp[ncols_dst][rows_per_cuda_block] = {{0.0f}};
+ float tmp_gate[ncols_dst][rows_per_cuda_block] = {{0.0f}};
+
+ const block_q8_1 * y = ((const block_q8_1 *) vy) + sample_y*stride_sample_y + channel_y*stride_channel_y;
+ const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row0*stride_row_x;
+
+ for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+ const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx
+
+ // x block quant index when casting the quants to int
+ const int kqs = vdr * (tid % (qi/vdr));
+
+#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
+ // start the next iterations' weight loads early
+ if constexpr (mmvq_should_prefetch(type)) {
+ constexpr int pf_dist = 2; // loop iterations, not blocks
+ const int kbx_pf = kbx + pf_dist*blocks_per_iter;
+ if (kbx_pf < blocks_per_row_x) {
+#pragma unroll
+ for (int i = 0; i < rows_per_cuda_block; ++i) {
+ const size_t off = (size_t)(kbx_offset + i*stride_row_x + kbx_pf) * ggml_cuda_type_traits<type>::bs;
+ mmvq_prefetch_l2((const char *) vx + off);
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ mmvq_prefetch_l2((const char *) vgate + off);
+ }
+ }
+ }
+ }
+ }
+#endif
+
+#pragma unroll
+ for (int j = 0; j < ncols_dst; ++j) {
+#pragma unroll
+ for (int i = 0; i < rows_per_cuda_block; ++i) {
+ tmp[j][i] += vec_dot_q_cuda(
+ vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ tmp_gate[j][i] += vec_dot_q_cuda(
+ vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
+ }
+ }
+ }
+ }
+ }
+
+ __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];
+ [[maybe_unused]] __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];
+
+ if (threadIdx.y > 0) {
+#pragma unroll
+ for (int j = 0; j < ncols_dst; ++j) {
+#pragma unroll
+ for (int i = 0; i < rows_per_cuda_block; ++i) {
+ tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i];
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ tmp_shared_gate[threadIdx.y-1][j][i][threadIdx.x] = tmp_gate[j][i];
+ }
+ }
+ }
+ }
+ }
+ __syncthreads();
+ if (threadIdx.y > 0) {
+ return;
+ }
+
+ dst += sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row0;
+
+ // sum up partial sums and write back result
+#pragma unroll
+ for (int j = 0; j < ncols_dst; ++j) {
+#pragma unroll
+ for (int i = 0; i < rows_per_cuda_block; ++i) {
+#pragma unroll
+ for (int l = 0; l < nwarps-1; ++l) {
+ tmp[j][i] += tmp_shared[l][j][i][threadIdx.x];
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ tmp_gate[j][i] += tmp_shared_gate[l][j][i][threadIdx.x];
+ }
+ }
+ }
+ tmp[j][i] = warp_reduce_sum<warp_size>(tmp[j][i]);
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ tmp_gate[j][i] = warp_reduce_sum<warp_size>(tmp_gate[j][i]);
+ }
+ }
+
+ if (threadIdx.x == i && (rows_per_cuda_block == 1 || uint32_t(row0 + i) < stride_col_dst)) {
+ float result = tmp[j][i];
+ if constexpr (has_fusion) {
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ result *= x_scales;
+ }
+ result += x_biases[j];
+ if (use_gate) {
+ float gate_value = tmp_gate[j][i];
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ gate_value *= gate_scales;
+ }
+ gate_value += gate_biases[j];
+ switch (active_glu) {
+ case GGML_GLU_OP_SWIGLU:
+ result *= ggml_cuda_op_silu_single(gate_value);
+ break;
+ case GGML_GLU_OP_GEGLU:
+ result *= ggml_cuda_op_gelu_single(gate_value);
+ break;
+ case GGML_GLU_OP_SWIGLU_OAI:
+ result = ggml_cuda_op_swiglu_oai_single(gate_value, result);
+ break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);
+ break;
+ default:
+ result = result * gate_value;
+ break;
+ }
+ }
+ }
+ dst[j*stride_col_dst + i] = ggml_cuda_cast<dst_t>(result);
+ }
+ }
+ }
+
+ if constexpr (!has_fusion) {
+ GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, glu_limit, gate_bias, x_bias, x_scale, gate_scale, tmp_gate);
+ }
+ if constexpr (type != GGML_TYPE_NVFP4) {
+ GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale, x_scales, gate_scales);
+ }
+}
+
+// Dedicated MoE multi-token kernel.
+// Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst)
+// Block: (warp_size, ncols_dst) - each warp handles one token independently.
+// No shared memory reduction needed since each warp works alone.
+template <ggml_type type, int c_rows_per_block, bool has_fusion = false, typename dst_t = float>
+__launch_bounds__(get_mmvq_mmid_max_batch_for_device<type>()*ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_q_moe(
+ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion,
+ dst_t * dst_ptr,
+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+ const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst,
+ const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
+ const uint32_t ncols_dst, const uint32_t ids_stride) {
+ const void * GGML_CUDA_RESTRICT vx = vx_ptr;
+ const void * GGML_CUDA_RESTRICT vy = vy_ptr;
+ const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
+ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;
+
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int qi = ggml_cuda_type_traits<type>::qi;
+ constexpr int vdr = get_vdr_mmvq(type);
+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+
+ constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
+
+ const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1;
+ if (shared_expert) {
+ vx = fusion.shared_up;
+ if constexpr (std::is_same_v<dst_t, float>) {
+ dst = fusion.shared_dst;
+ }
+ stride_col_dst = fusion.shared_stride_col_dst;
+ }
+
+ // fuse gate, bias, scales, and glu_op into the up projection
+ bool use_gate = false;
+ const void * vgate = nullptr;
+ const float * x_bias = nullptr;
+ const float * gate_bias = nullptr;
+ const float * x_scale = nullptr;
+ const float * gate_scale = nullptr;
+ ggml_glu_op active_glu = GGML_GLU_OP_SWIGLU;
+ float glu_limit = 0.0f;
+
+ if constexpr (has_fusion) {
+ use_gate = fusion.gate != nullptr;
+ vgate = shared_expert ? fusion.shared_gate : fusion.gate;
+ x_bias = (const float *) fusion.x_bias;
+ gate_bias = (const float *) fusion.gate_bias;
+ active_glu = fusion.glu_op;
+ glu_limit = fusion.glu_limit;
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ x_scale = (const float *) fusion.x_scale;
+ gate_scale = (const float *) fusion.gate_scale;
+ }
+ }
+
+ const uint32_t token_idx = threadIdx.y;
+ const int row0 = c_rows_per_block*blockIdx.x;
+ const int blocks_per_row_x = ncols_x / qk;
+ constexpr int blocks_per_iter = vdr * warp_size / qi;
+
+ const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y;
+
+ if (token_idx >= ncols_dst) {
+ return;
+ }
+
+ ggml_cuda_pdl_sync();
+ const uint32_t channel_x = shared_expert ? 0 : ids[channel_dst + token_idx * ids_stride];
+ const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y);
+
+ const block_q8_1 * y = ((const block_q8_1 *) vy) + channel_y*stride_channel_y + token_idx*stride_col_y;
+ const int kbx_offset = channel_x*stride_channel_x + row0*stride_row_x;
+
+ // partial sum for each thread
+ float tmp[c_rows_per_block] = {0.0f};
+ float tmp_gate[c_rows_per_block] = {0.0f};
+
+ for (int kbx = threadIdx.x / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+ const int kby = kbx * (qk/QK8_1);
+ const int kqs = vdr * (threadIdx.x % (qi/vdr));
+
+#pragma unroll
+ for (int i = 0; i < c_rows_per_block; ++i) {
+ tmp[i] += vec_dot_q_cuda(vx, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs);
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ tmp_gate[i] += vec_dot_q_cuda(vgate, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs);
+ }
+ }
+ }
+ }
+
+ ggml_cuda_pdl_lc();
+
+ // Warp-level reduction only - no shared memory needed
+#pragma unroll
+ for (int i = 0; i < c_rows_per_block; ++i) {
+ tmp[i] = warp_reduce_sum<warp_size>(tmp[i]);
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ tmp_gate[i] = warp_reduce_sum<warp_size>(tmp_gate[i]);
+ }
+ }
+ }
+
+ // Write results
+ if (threadIdx.x < c_rows_per_block && (c_rows_per_block == 1 || uint32_t(row0 + threadIdx.x) < nrows_x)) {
+ float result = tmp[threadIdx.x];
+ if constexpr (has_fusion) {
+ const uint32_t bias_idx = channel_x*stride_channel_dst + row0 + threadIdx.x;
+
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (x_scale) {
+ result *= x_scale[channel_x];
+ }
+ }
+ if (x_bias) {
+ result += x_bias[bias_idx];
+ }
+ if (use_gate) {
+ float gate_value = tmp_gate[threadIdx.x];
+ if constexpr (type == GGML_TYPE_NVFP4) {
+ if (gate_scale) {
+ gate_value *= gate_scale[channel_x];
+ }
+ }
+ if (gate_bias) {
+ gate_value += gate_bias[bias_idx];
+ }
+ switch (active_glu) {
+ case GGML_GLU_OP_SWIGLU:
+ result *= ggml_cuda_op_silu_single(gate_value);
+ break;
+ case GGML_GLU_OP_GEGLU:
+ result *= ggml_cuda_op_gelu_single(gate_value);
+ break;
+ case GGML_GLU_OP_SWIGLU_OAI:
+ result = ggml_cuda_op_swiglu_oai_single(gate_value, result);
+ break;
+ case GGML_GLU_OP_SWIGLU_CLAMP:
+ result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);
+ break;
+ default:
+ result = result * gate_value;
+ break;
+ }
+ }
+ }
+ dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = ggml_cuda_cast<dst_t>(result);
+ }
+
+ if constexpr (!has_fusion) {
+ GGML_UNUSED_VARS(use_gate, tmp_gate, vgate, x_bias, gate_bias, active_glu, glu_limit, x_scale, gate_scale);
+ } else if constexpr (type != GGML_TYPE_NVFP4) {
+ GGML_UNUSED_VARS(x_scale, gate_scale);
+ }
+}
+
+template<ggml_type type>
+static std::pair<dim3, dim3> calc_launch_params(
+ const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens,
+ const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false, const bool halve_iters = false) {
+ const int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);
+ const int rpb = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);
+ const int64_t nblocks = (nrows_x + rpb - 1) / rpb;
+ const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens);
+ const dim3 block_dims(warp_size, nwarps, 1);
+ return {block_nums, block_dims};
+}
+
+template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false, typename dst_t>
+static void mul_mat_vec_q_switch_fusion(
+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,
+ const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
+ const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
+ const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,
+ const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared,
+ const uint32_t ids_stride, cudaStream_t stream) {
+
+ const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||
+ fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
+ if constexpr (c_ncols_dst == 1) {
+ if (has_fusion) {
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
+ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters, dst_t>, launch_params,
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
+ return;
+ }
+ }
+
+ GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1");
+
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
+ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters, dst_t>, launch_params,
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
+}
+
+template <ggml_type type, typename dst_t>
+static void mul_mat_vec_q_moe_launch(
+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+ const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,
+ const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
+ const uint32_t ncols_dst, const uint32_t ids_stride,
+ const int warp_size, const int nchannels_dst, cudaStream_t stream) {
+
+ constexpr int rows_per_block = 2; // 2 gives best perf based on tuning
+ const int64_t nblocks_rows = (nrows_x + rows_per_block - 1) / rows_per_block;
+ const dim3 block_nums(nblocks_rows, nchannels_dst + (fusion.shared_up != nullptr));
+ const dim3 block_dims(warp_size, ncols_dst);
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
+
+ const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||
+ fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
+
+ if (has_fusion) {
+ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true, dst_t>, launch_params,
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
+ stride_row_x, stride_col_y, stride_col_dst,
+ stride_channel_x, stride_channel_y, stride_channel_dst,
+ ncols_dst, ids_stride);
+ } else {
+ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false, dst_t>, launch_params,
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
+ stride_row_x, stride_col_y, stride_col_dst,
+ stride_channel_x, stride_channel_y, stride_channel_dst,
+ ncols_dst, ids_stride);
+ }
+}
+
+template <ggml_type type, typename dst_t>
+static void mul_mat_vec_q_switch_ncols_dst(
+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
+ const int ncols_x, const int nrows_x, const int ncols_dst,
+ const int stride_row_x, const int stride_col_y, const int stride_col_dst,
+ const int nchannels_x, const int nchannels_y, const int nchannels_dst,
+ const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
+ const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
+ const int ids_stride, cudaStream_t stream) {
+
+ GGML_ASSERT(ncols_x % ggml_blck_size(type) == 0);
+ GGML_ASSERT(ncols_dst <= MMVQ_MAX_BATCH_SIZE);
+
+ const uint3 nchannels_y_fd = ids ? init_fastdiv_values(nchannels_y) : make_uint3(0, 0, 0);
+ const uint3 channel_ratio_fd = ids ? make_uint3(0, 0, 0) : init_fastdiv_values(nchannels_dst / nchannels_x);
+ const uint3 sample_ratio_fd = init_fastdiv_values(nsamples_dst / nsamples_x);
+
+ const int device = ggml_cuda_get_device();
+ const int cc = ggml_cuda_info().devices[device].cc;
+ const int warp_size = ggml_cuda_info().devices[device].warp_size;
+ const mmvq_parameter_table_id table_id = get_device_table_id(cc);
+
+ const bool has_ids = ids != nullptr;
+
+ // How the K loop divides up at the baseline block width, both decisions below use these.
+ constexpr int qk = ggml_cuda_type_traits<type>::qk;
+ constexpr int qi = ggml_cuda_type_traits<type>::qi;
+ constexpr int vdr = get_vdr_mmvq(type);
+ const int blocks_per_row_x = ncols_x / qk;
+ const int blocks_per_iter_1warp = vdr * warp_size / qi;
+
+ const auto should_use_small_k = [&](int c_ncols_dst) {
+ // When K is small, increase rows_per_block to match nwarps so each warp has more work to do
+ // Trigger when the full thread block covers all K blocks in a single loop iteration and few threads remain idle.
+ const int nwarps = calc_nwarps(type, c_ncols_dst, table_id);
+ bool use = nwarps > 1 && blocks_per_row_x < nwarps * blocks_per_iter_1warp;
+
+ constexpr std::array<ggml_type, 2> iq_slow_turing = {
+ GGML_TYPE_IQ3_XXS,
+ GGML_TYPE_IQ3_S,
+ };
+ constexpr std::array<ggml_type, 8> iq_slow_other = {
+ GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS,
+ GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS,
+ };
+ constexpr std::array<ggml_type, 3> slow_pascal = {
+ GGML_TYPE_IQ3_S,
+ GGML_TYPE_Q2_K,
+ GGML_TYPE_Q3_K,
+ };
+
+ const bool is_nvidia_turing_plus = GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_TURING;
+ const bool is_nvidia_pascal_older = GGML_CUDA_CC_IS_NVIDIA(cc) && cc < GGML_CUDA_CC_VOLTA;
+
+ if (is_nvidia_turing_plus) {
+ if (ncols_dst == 1 &&
+ std::find(iq_slow_turing.begin(), iq_slow_turing.end(), type) != iq_slow_turing.end()) {
+ use = false;
+ }
+ } else if ((ncols_dst == 1 && std::find(iq_slow_other.begin(), iq_slow_other.end(), type) != iq_slow_other.end()) ||
+ (is_nvidia_pascal_older && std::find(slow_pascal.begin(), slow_pascal.end(), type) != slow_pascal.end()) ||
+ GGML_CUDA_CC_IS_RDNA(cc)) {
+ use = false;
+ }
+
+ return use;
+ };
+
+ // Whether doubling nwarps pays off on the ncols_dst == 1 path, where K sets the K loop trip count.
+ const auto should_halve_iters = [&] {
+ if (table_id != MMVQ_PARAMETERS_GB10) {
+ return false;
+ }
+
+ // Expert rows are gathered per token, so a wider block adds reduction work without reuse.
+ if (has_ids) {
+ return false;
+ }
+
+ const int blocks_per_iter = calc_nwarps(type, 1, table_id) * blocks_per_iter_1warp;
+ const int iters = (blocks_per_row_x + blocks_per_iter - 1) / blocks_per_iter;
+ const int iters_wide = (blocks_per_row_x + blocks_per_iter * 2 - 1) / (blocks_per_iter * 2);
+
+ // An odd trip count leaves half the wider block idle for its last iteration, that tail is
+ // only affordable once the loop is long enough to dilute it to an eighth of the work (observation).
+ const int idle = iters_wide * 2 - iters;
+
+ return idle * 8 <= iters_wide * 2;
+ };
+
+ if (has_ids && ncols_dst > 1) {
+ // Multi-token MUL_MAT_ID path - dedicated MoE kernel
+ mul_mat_vec_q_moe_launch<type>(
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, nrows_x,
+ stride_row_x, stride_col_y, stride_col_dst,
+ stride_channel_x, stride_channel_y, stride_channel_dst,
+ ncols_dst, ids_stride, warp_size, nchannels_dst, stream);
+ return;
+ }
+
+ switch (ncols_dst) {
+ case 1: {
+ // static, else MSVC lambda capture breaks the constexpr uses below
+ static constexpr int c_ncols_dst = 1;
+
+ // Tag types keep the flags compile-time, so __launch_bounds__ matches what is launched.
+ const auto launch = [&](auto small_k_tag, auto halve_iters_tag) {
+ constexpr bool c_small_k = decltype(small_k_tag)::value;
+ // Types the table does not promote would compile a second, identical kernel.
+ constexpr bool c_promoted =
+ calc_nwarps(type, c_ncols_dst, MMVQ_PARAMETERS_GB10, false, true) !=
+ calc_nwarps(type, c_ncols_dst, MMVQ_PARAMETERS_GB10, false, false);
+
+ constexpr bool c_halve_iters = decltype(halve_iters_tag)::value && c_promoted;
+
+ const std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst + (fusion.shared_up != nullptr),
+ nsamples_dst, warp_size, table_id, c_small_k, c_halve_iters);
+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst, c_small_k, c_halve_iters>(
+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd,
+ stride_sample_x, stride_sample_y, stride_sample_dst, dims.first, dims.second, 0, ids_stride,
+ stream);
+ };
+
+ if (should_use_small_k(c_ncols_dst)) {
+ launch(std::true_type{}, std::false_type{});
+ } else if (should_halve_iters()) {
+ launch(std::false_type{}, std::true_type{});
+ } else {
+ launch(std::false_type{}, std::false_type{});
+ }
+ } break;
+ case 2: {
+ constexpr int c_ncols_dst = 2;
+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
+ dims.first, dims.second, 0, ids_stride, stream);
+ } break;
+ case 3: {
+ constexpr int c_ncols_dst = 3;
+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
+ dims.first, dims.second, 0, ids_stride, stream);
+ } break;
+ case 4: {
+ constexpr int c_ncols_dst = 4;
+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
+ dims.first, dims.second, 0, ids_stride, stream);
+ } break;
+ case 5: {
+ constexpr int c_ncols_dst = 5;
+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
+ dims.first, dims.second, 0, ids_stride, stream);
+ } break;
+ case 6: {
+ constexpr int c_ncols_dst = 6;
+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
+ dims.first, dims.second, 0, ids_stride, stream);
+ } break;
+ case 7: {
+ constexpr int c_ncols_dst = 7;
+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
+ dims.first, dims.second, 0, ids_stride, stream);
+ } break;
+ case 8: {
+ constexpr int c_ncols_dst = 8;
+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
+ dims.first, dims.second, 0, ids_stride, stream);
+ } break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+template <typename dst_t>
+static void mul_mat_vec_q_switch_type(
+ const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,
+ const int ncols_x, const int nrows_x, const int ncols_dst,
+ const int stride_row_x, const int stride_col_y, const int stride_col_dst,
+ const int nchannels_x, const int nchannels_y, const int nchannels_dst,
+ const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
+ const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
+ const int ids_stride, cudaStream_t stream) {
+ switch (type_x) {
+ case GGML_TYPE_Q1_0:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q1_0>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q2_0:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q2_0>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q4_0:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_0>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q4_1:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_1>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q5_0:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_0>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q5_1:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_1>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q8_0:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q8_0>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_MXFP4:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_MXFP4>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_NVFP4:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_NVFP4>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q2_K:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q2_K>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q3_K:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q3_K>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q4_K:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_K>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q5_K:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_K>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_Q6_K:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q6_K>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ2_XXS:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_XXS>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ2_XS:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_XS>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ2_S:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_S>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ3_XXS:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ3_XXS>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ1_S:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ1_S>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ1_M:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ1_M>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ4_NL:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ4_NL>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ4_XS:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ4_XS>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ case GGML_TYPE_IQ3_S:
+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ3_S>
+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+void ggml_cuda_mul_mat_vec_q(
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
+ const ggml_cuda_mm_fusion_args_host * fusion, const void * src1_q8_1_pre) {
+ // FreeToken: activations and results may also be bf16 or fp16
+ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);
+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+
+ cudaStream_t stream = ctx.stream();
+
+ const size_t ts_src0 = ggml_type_size(src0->type);
+ const size_t ts_src1 = ggml_type_size(src1->type);
+ const size_t ts_dst = ggml_type_size(dst->type);
+
+ GGML_ASSERT( nb00 == ts_src0);
+ GGML_ASSERT( nb10 == ts_src1);
+ GGML_ASSERT( nb0 == ts_dst);
+ GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));
+
+ GGML_ASSERT(!ids || ne12 <= MMVQ_MAX_BATCH_SIZE);
+
+ const float * src1_d = (const float *) src1->data;
+ const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr;
+ float * dst_d = (float *) dst->data;
+
+ ggml_cuda_mm_fusion_args_device fusion_local{};
+
+ if (fusion) {
+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+ GGML_ASSERT( !ids || dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc));
+ GGML_ASSERT( ids || dst->ne[1] == 1);
+ // Scale fusion is only allowed for NVFP4 currently as the cost of checking this at run-time in the prologue is
+ // non-negligible for some models such as gpt-oss-20b
+ GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_NVFP4);
+
+ if (fusion->shared_up) {
+ GGML_ASSERT(ids && fusion->gate && fusion->shared_gate && fusion->shared_dst);
+ GGML_ASSERT(!fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale);
+ GGML_ASSERT(ne11 == 1 && ne03 == 1 && ne13 == 1);
+ GGML_ASSERT(fusion->shared_up->type == src0->type && fusion->shared_gate->type == src0->type);
+ GGML_ASSERT(ggml_are_same_shape(fusion->shared_up, fusion->shared_gate));
+ GGML_ASSERT(ggml_is_contiguous(fusion->shared_up) && ggml_is_contiguous(fusion->shared_gate));
+ GGML_ASSERT(fusion->shared_up->ne[0] == ne00 && fusion->shared_up->ne[1] == ne01);
+ GGML_ASSERT(fusion->shared_up->nb[1] == nb01 && ggml_is_matrix(fusion->shared_up));
+ GGML_ASSERT(fusion->shared_dst->type == GGML_TYPE_F32 && ggml_is_contiguous(fusion->shared_dst));
+ GGML_ASSERT(fusion->shared_dst->ne[0] == ne0 && fusion->shared_dst->ne[1] == ne2);
+ fusion_local.shared_up = fusion->shared_up->data;
+ fusion_local.shared_gate = fusion->shared_gate->data;
+ fusion_local.shared_dst = (float *) fusion->shared_dst->data;
+ fusion_local.shared_stride_col_dst = fusion->shared_dst->nb[1] / ts_dst;
+ }
+
+ if (fusion->x_bias) {
+ GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32);
+ GGML_ASSERT(fusion->x_bias->ne[0] == dst->ne[0]);
+ GGML_ASSERT(!ids || fusion->x_bias->ne[1] == src0->ne[2]);
+ fusion_local.x_bias = fusion->x_bias->data;
+ }
+ if (fusion->gate) {
+ GGML_ASSERT(fusion->gate->type == src0->type && ggml_are_same_stride(fusion->gate, src0));
+ fusion_local.gate = fusion->gate->data;
+ }
+ if (fusion->gate_bias) {
+ GGML_ASSERT(fusion->gate_bias->type == GGML_TYPE_F32);
+ GGML_ASSERT(fusion->gate_bias->ne[0] == dst->ne[0]);
+ GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);
+ fusion_local.gate_bias = fusion->gate_bias->data;
+ }
+ if (fusion->x_scale) {
+ GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_is_contiguous(fusion->x_scale));
+ GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1));
+ fusion_local.x_scale = fusion->x_scale->data;
+ }
+ if (fusion->gate_scale) {
+ GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale));
+ GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));
+ fusion_local.gate_scale = fusion->gate_scale->data;
+ }
+ fusion_local.glu_op = fusion->glu_op;
+ fusion_local.glu_limit = fusion->glu_limit;
+ }
+
+ // If src0 is a temporary compute buffer, clear any potential padding.
+ if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
+ const size_t size_data = ggml_nbytes(src0);
+ const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);
+ if (size_alloc > size_data) {
+ GGML_ASSERT(ggml_is_contiguously_allocated(src0));
+ GGML_ASSERT(!src0->view_src);
+ CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));
+ }
+ }
+
+ const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);
+ // FreeToken: src1_q8_1_pre is src1 already quantized by quantize_row_q8_1_cuda with this padding and layout
+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool());
+ const void * src1_q8 = src1_q8_1_pre;
+ if (src1_q8 == nullptr) {
+ src1_q8_1.alloc(ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);
+ src1_q8 = src1_q8_1.get();
+ const int64_t s11 = src1->nb[1] / ts_src1;
+ const int64_t s12 = src1->nb[2] / ts_src1;
+ const int64_t s13 = src1->nb[3] / ts_src1;
+ if (src1->type == GGML_TYPE_BF16) {
+ quantize_row_q8_1_cuda((const nv_bfloat16 *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
+ } else if (src1->type == GGML_TYPE_F16) {
+ quantize_row_q8_1_cuda((const half *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
+ } else {
+ quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
+ }
+ }
+
+ const int64_t s01 = src0->nb[1] / ts_src0;
+ const int64_t s11 = ne10_padded / QK8_1;
+ const int64_t s1 = dst->nb[1] / ts_dst;
+ const int64_t s02 = src0->nb[2] / ts_src0;
+ const int64_t s2 = dst->nb[2] / ts_dst;
+ const int64_t s03 = src0->nb[3] / ts_src0;
+ const int64_t s3 = dst->nb[3] / ts_dst;
+
+ const int64_t s12 = ne11*s11;
+ const int64_t s13 = ne12*s12;
+
+ // For MUL_MAT_ID the memory layout is different than for MUL_MAT:
+ const int64_t ncols_dst = ids ? ne2 : ne1;
+ const int64_t nchannels_y = ids ? ne11 : ne12;
+ const int64_t nchannels_dst = ids ? ne1 : ne2;
+ const int64_t stride_col_dst = ids ? s2 : s1;
+ const int64_t stride_col_y = ids ? s12 : s11;
+ const int64_t stride_channel_dst = ids ? s1 : s2;
+ const int64_t stride_channel_y = ids ? s11 : s12;
+
+ const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
+
+ const auto launch = [&](auto * dst_ptr) {
+ mul_mat_vec_q_switch_type(
+ src0->data, src0->type, src1_q8, ids_d, fusion_local, dst_ptr, ne00,
+ ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
+ ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
+ ne03, ne3, s03, s13, s3, ids_stride, stream);
+ };
+ if (dst->type == GGML_TYPE_BF16) {
+ // FreeToken: the fused shared expert writes an fp32 shared_dst
+ GGML_ASSERT(!fusion || !fusion->shared_up);
+ launch((nv_bfloat16 *) dst->data);
+ } else if (dst->type == GGML_TYPE_F16) {
+ GGML_ASSERT(!fusion || !fusion->shared_up);
+ launch((half *) dst->data);
+ } else {
+ launch(dst_d);
+ }
+}
+
+void ggml_cuda_op_mul_mat_vec_q(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream) {
+
+ const int64_t ne00 = src0->ne[0];
+ const int64_t row_diff = row_high - row_low;
+
+ const int64_t ne10 = src1->ne[0];
+ GGML_ASSERT(ne10 % QK8_1 == 0);
+
+ const int64_t ne0 = dst->ne[0];
+
+ int id = ggml_cuda_get_device();
+
+ // the main device has a larger memory buffer to hold the results from all GPUs
+ // nrows_dst == nrows of the matrix that the kernel writes into
+ const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;
+
+ const int stride_row_x = ne00 / ggml_blck_size(src0->type);
+ const int stride_col_y = src1_padded_row_size / QK8_1;
+
+ ggml_cuda_mm_fusion_args_device fusion_local{};
+ mul_mat_vec_q_switch_type(
+ src0_dd_i, src0->type, src1_ddq_i, nullptr, fusion_local, dst_dd_i, ne00, row_diff, src1_ncols, stride_row_x, stride_col_y, nrows_dst,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, stream);
+
+ GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_ncols, src1_padded_row_size);
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cuh
new file mode 100644
index 00000000..6973780c
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cuh
@@ -0,0 +1,20 @@
+#include "common.cuh"
+
+#define MMVQ_MAX_BATCH_SIZE 8 // Max. batch size for which to use MMVQ kernels.
+
+bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11);
+
+// Returns the maximum batch size for which MMVQ should be used for MUL_MAT_ID,
+// based on the quantization type and GPU architecture (compute capability).
+int get_mmvq_mmid_max_batch(ggml_type type, int cc);
+
+// FreeToken: src1_q8_1_pre skips the internal quantization of src1
+void ggml_cuda_mul_mat_vec_q(ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr,
+ const void * src1_q8_1_pre = nullptr);
+
+void ggml_cuda_op_mul_mat_vec_q(
+ ggml_backend_cuda_context & ctx,
+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
+ const int64_t src1_padded_row_size, cudaStream_t stream);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cu
new file mode 100644
index 00000000..46c9f861
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cu
@@ -0,0 +1,729 @@
+#include "quantize.cuh"
+#include "convert.cuh"
+#include <cstdint>
+
+#if defined(BLACKWELL_MMA_AVAILABLE)
+// this maps to 256-bit loads in PTX on supported devices,
+// and otherwise falls back to 2 128-bit loads
+struct __builtin_align__(32) float8 {
+ float x; float y; float z; float w;
+ float p; float q; float r; float s;
+};
+
+#if CUDART_VERSION >= 12080
+static __device__ __forceinline__ float nvfp4_native_scale_error(
+ const float vals[QK_NVFP4_SUB], const float inv_col_scale, const float inv_scale, const float scale) {
+ const float scale_dequant = 2.0f * scale;
+ float err = 0.0f;
+
+#pragma unroll
+ for (int k = 0; k < QK_NVFP4_SUB; k += 4) {
+ const float v0 = vals[k + 0] * inv_col_scale;
+ const float v1 = vals[k + 1] * inv_col_scale;
+ const float v2 = vals[k + 2] * inv_col_scale;
+ const float v3 = vals[k + 3] * inv_col_scale;
+
+ const __nv_fp4x4_e2m1 q(make_float4(v0 * inv_scale, v1 * inv_scale, v2 * inv_scale, v3 * inv_scale));
+ const __nv_fp4x4_storage_t q_storage = q.__x;
+ const __nv_fp4x2_storage_t q_lo = static_cast<__nv_fp4x2_storage_t>(q_storage);
+ const __nv_fp4x2_storage_t q_hi = static_cast<__nv_fp4x2_storage_t>(q_storage >> 8U);
+
+ const __half2_raw hraw2_lo = __nv_cvt_fp4x2_to_halfraw2(q_lo, __NV_E2M1);
+ const __half2_raw hraw2_hi = __nv_cvt_fp4x2_to_halfraw2(q_hi, __NV_E2M1);
+ const __half2 h2_lo = static_cast<__half2>(hraw2_lo);
+ const __half2 h2_hi = static_cast<__half2>(hraw2_hi);
+ const float2 dq_lo = __half22float2(h2_lo);
+ const float2 dq_hi = __half22float2(h2_hi);
+
+ const float err0 = fabsf(v0) - fabsf(dq_lo.x) * scale_dequant;
+ const float err1 = fabsf(v1) - fabsf(dq_lo.y) * scale_dequant;
+ const float err2 = fabsf(v2) - fabsf(dq_hi.x) * scale_dequant;
+ const float err3 = fabsf(v3) - fabsf(dq_hi.y) * scale_dequant;
+
+ err = fmaf(err0, err0, err);
+ err = fmaf(err1, err1, err);
+ err = fmaf(err2, err2, err);
+ err = fmaf(err3, err3, err);
+ }
+
+ return err;
+}
+#endif // CUDART_VERSION >= 12080
+#endif // defined(BLACKWELL_MMA_AVAILABLE)
+
+template <typename src_t>
+__launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1)
+static __global__ void quantize_q8_1(
+ const src_t * x_ptr, void * vy_ptr,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const uint32_t ne1, const uint3 ne2) {
+ ggml_cuda_pdl_lc();
+ const src_t * GGML_CUDA_RESTRICT x = x_ptr;
+ void * GGML_CUDA_RESTRICT vy = vy_ptr;
+ const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
+
+ if (i0 >= ne0) {
+ return;
+ }
+
+ const int64_t i3 = fastdiv(blockIdx.z, ne2);
+ const int64_t i2 = blockIdx.z - i3*ne2.z;
+ const int64_t i1 = blockIdx.y;
+
+ const int64_t & i00 = i0;
+ const int64_t & i01 = i1;
+ const int64_t & i02 = i2;
+ const int64_t & i03 = i3;
+
+ const int64_t i_cont = ((i3*ne2.z + i2) * ne1 + i1) * ne0 + i0;
+
+ block_q8_1 * y = (block_q8_1 *) vy;
+
+ const int64_t ib = i_cont / QK8_1; // block index
+ const int64_t iqs = i_cont % QK8_1; // quant index
+
+ ggml_cuda_pdl_sync();
+ const float xi = i0 < ne00 ? ggml_cuda_cast<float>(x[i03*s03 + i02*s02 + i01*s01 + i00]) : 0.0f;
+ float amax = fabsf(xi);
+ float sum = xi;
+
+ amax = warp_reduce_max<QK8_1>(amax);
+ sum = warp_reduce_sum<QK8_1>(sum);
+
+ const float d = amax / 127.0f;
+ const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
+
+ y[ib].qs[iqs] = q;
+
+ if (iqs > 0) {
+ return;
+ }
+
+ y[ib].ds = make_half2(d, sum);
+}
+
+__device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) {
+ if (!(amax > 0.0f)) {
+ return 0;
+ }
+
+ // FP4 E2M1: max exponent (unbiased) is 2.
+ constexpr int FP4_E2M1_EMAX = 2;
+
+ const float e = log2f(amax);
+
+ // "even" -> round-to-nearest integer, ties-to-even
+ const int e_int = __float2int_rn(e);
+
+ const int shared_exp = e_int - FP4_E2M1_EMAX;
+
+ int biased = shared_exp + 127;
+
+ biased = max(biased, 0);
+ biased = min(biased, 254);
+
+ return static_cast<uint8_t>(biased);
+}
+
+// scatter: grid over tokens, quantize once, write to all the token's compact rows
+template <bool scatter, bool use_aligned_float8>
+static __global__ void quantize_mmq_nvfp4(
+ const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, float * __restrict__ scale,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int n_expert_used) {
+#if defined(BLACKWELL_MMA_AVAILABLE)
+
+ int64_t base_idx;
+ if constexpr (scatter) {
+ base_idx = (int64_t) blockIdx.x * s02; // one physical row per token
+ } else {
+ const int64_t i2 = blockIdx.y % ne2;
+ const int64_t i3 = blockIdx.y / ne2;
+ const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x;
+ base_idx = i3 * s03 + i2 * s02 + i01 * s01;
+ }
+ const float * __restrict__ x_row = x + base_idx;
+
+ float amax = 0.0f;
+ if constexpr (use_aligned_float8) {
+ for (int64_t i0 = 8 * threadIdx.x; i0 < ne00; i0 += 8 * blockDim.x) {
+ const float * x_base = x_row + i0;
+ const float8 v = reinterpret_cast<const float8 *>(x_base)[0];
+ amax = fmaxf(amax, fabsf(v.x));
+ amax = fmaxf(amax, fabsf(v.y));
+ amax = fmaxf(amax, fabsf(v.z));
+ amax = fmaxf(amax, fabsf(v.w));
+ amax = fmaxf(amax, fabsf(v.p));
+ amax = fmaxf(amax, fabsf(v.q));
+ amax = fmaxf(amax, fabsf(v.r));
+ amax = fmaxf(amax, fabsf(v.s));
+ }
+ } else {
+ for (int64_t i0 = threadIdx.x; i0 < ne00; i0 += blockDim.x) {
+ amax = fmaxf(amax, fabsf(x_row[i0]));
+ }
+ }
+
+ amax = warp_reduce_max<WARP_SIZE>(amax);
+
+ __shared__ float warp_amax[CUDA_QUANTIZE_BLOCK_SIZE_MMQ / WARP_SIZE];
+ const int lane = threadIdx.x % WARP_SIZE;
+ const int warp = threadIdx.x / WARP_SIZE;
+
+ if (lane == 0) {
+ warp_amax[warp] = amax;
+ }
+ __syncthreads();
+
+ if (warp == 0) {
+ amax = threadIdx.x < int(CUDA_QUANTIZE_BLOCK_SIZE_MMQ / WARP_SIZE) ? warp_amax[lane] : 0.0f;
+ amax = warp_reduce_max<WARP_SIZE>(amax);
+ if (lane == 0) {
+ warp_amax[0] = amax / (6.0f * 448.0f);
+ if constexpr (scatter) {
+#pragma unroll
+ for (int slot = 0; slot < n_expert_used; ++slot) {
+ const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];
+ scale[i] = warp_amax[0];
+ }
+ } else {
+ scale[blockIdx.y * ne1 + blockIdx.x] = warp_amax[0];
+ }
+ }
+ }
+ __syncthreads();
+
+ block_fp4_mmq * y = (block_fp4_mmq *) vy;
+ const int64_t n_subblocks = (ne0 + QK_NVFP4_SUB - 1) / QK_NVFP4_SUB;
+
+ for (int64_t isb = threadIdx.x; isb < n_subblocks; isb += blockDim.x) {
+ const int64_t i0_base = isb * QK_NVFP4_SUB;
+ const int64_t k_block = i0_base / QK_FP4_MMQ;
+ const int sub = (i0_base % QK_FP4_MMQ) / QK_NVFP4_SUB;
+
+ const float row_scale = warp_amax[0];
+ const float inv_col_scale = row_scale > 0.0f ? 1.0f / row_scale : 0.0f;
+
+ float vals[QK_NVFP4_SUB];
+ if constexpr (use_aligned_float8) {
+ const float * x_base = x_row + i0_base;
+ const float8 v0 = i0_base + 7 < ne00 ? reinterpret_cast<const float8 *>(x_base)[0] : float8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
+ const float8 v1 = i0_base + 15 < ne00 ? reinterpret_cast<const float8 *>(x_base + 8)[0] : float8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
+ vals[0] = v0.x; vals[1] = v0.y; vals[2] = v0.z; vals[3] = v0.w;
+ vals[4] = v0.p; vals[5] = v0.q; vals[6] = v0.r; vals[7] = v0.s;
+ vals[8] = v1.x; vals[9] = v1.y; vals[10] = v1.z; vals[11] = v1.w;
+ vals[12] = v1.p; vals[13] = v1.q; vals[14] = v1.r; vals[15] = v1.s;
+ } else {
+#pragma unroll
+ for (int k = 0; k < QK_NVFP4_SUB; ++k) {
+ const int64_t i00 = i0_base + k;
+ vals[k] = i00 < ne00 ? x_row[i00] : 0.0f;
+ }
+ }
+
+ uint32_t q0 = 0;
+ uint32_t q1 = 0;
+
+ float amax_sub = 0.0f;
+#pragma unroll
+ for (int k = 0; k < QK_NVFP4_SUB; ++k) {
+ amax_sub = fmaxf(amax_sub, fabsf(vals[k] * inv_col_scale));
+ }
+
+ static constexpr int test_offsets[5] = { 0, -1, 1, -2, 2 };
+ const int first_fp8_code = (int) ggml_cuda_fp32_to_ue4m3(amax_sub / 6.0f);
+
+ uint8_t fp8_code = (uint8_t) first_fp8_code;
+ float subblock_scale = ggml_cuda_ue4m3_to_fp32(fp8_code);
+ float inv_scale_err = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f;
+#if CUDART_VERSION >= 12080
+ float best_err = nvfp4_native_scale_error(vals, inv_col_scale, inv_scale_err, subblock_scale);
+#else
+ float best_err = 0.0f;
+#pragma unroll
+ for (int k = 0; k < QK_NVFP4_SUB; ++k) {
+ const float v = vals[k] * inv_col_scale;
+ const uint8_t q = ggml_cuda_float_to_fp4_e2m1(v, inv_scale_err);
+ const float err_diff = fabsf(v) - fabsf(kvalues_fp4[q & 0x7]) * subblock_scale;
+ best_err = fmaf(err_diff, err_diff, best_err);
+ }
+#endif // CUDART_VERSION >= 12080
+
+#pragma unroll
+ for (int i = 1; i < 5; ++i) {
+ const int test_code = first_fp8_code + test_offsets[i];
+ if (test_code < 0 || test_code > 0x7e) {
+ continue;
+ }
+
+ const float test_scale = ggml_cuda_ue4m3_to_fp32((uint8_t) test_code);
+ const float test_inv_scale = test_scale > 0.0f ? 0.5f / test_scale : 0.0f;
+#if CUDART_VERSION >= 12080
+ const float cur_err = nvfp4_native_scale_error(vals, inv_col_scale, test_inv_scale, test_scale);
+#else
+ float cur_err = 0.0f;
+#pragma unroll
+ for (int k = 0; k < QK_NVFP4_SUB; ++k) {
+ const float v = vals[k] * inv_col_scale;
+ const uint8_t q = ggml_cuda_float_to_fp4_e2m1(v, test_inv_scale);
+ const float err_diff = fabsf(v) - fabsf(kvalues_fp4[q & 0x7]) * test_scale;
+ cur_err = fmaf(err_diff, err_diff, cur_err);
+ }
+#endif // CUDART_VERSION >= 12080
+
+ if (cur_err < best_err) {
+ best_err = cur_err;
+ fp8_code = (uint8_t) test_code;
+ subblock_scale = test_scale;
+ }
+ }
+#if CUDART_VERSION >= 12080
+ const float inv_scale = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f;
+ const float s = inv_col_scale * inv_scale;
+
+ __nv_fp4x4_e2m1 q0_lo(make_float4(vals[0] * s, vals[8] * s, vals[1] * s, vals[9] * s));
+ __nv_fp4x4_e2m1 q0_hi(make_float4(vals[2] * s, vals[10] * s, vals[3] * s, vals[11] * s));
+ __nv_fp4x4_e2m1 q1_lo(make_float4(vals[4] * s, vals[12] * s, vals[5] * s, vals[13] * s));
+ __nv_fp4x4_e2m1 q1_hi(make_float4(vals[6] * s, vals[14] * s, vals[7] * s, vals[15] * s));
+
+ const char2 q0_lo_c = *reinterpret_cast<char2 *>(&q0_lo);
+ const char2 q0_hi_c = *reinterpret_cast<char2 *>(&q0_hi);
+ const char2 q1_lo_c = *reinterpret_cast<char2 *>(&q1_lo);
+ const char2 q1_hi_c = *reinterpret_cast<char2 *>(&q1_hi);
+
+ q0 = uint32_t(uint8_t(q0_lo_c.x)) | (uint32_t(uint8_t(q0_lo_c.y)) << 8) |
+ (uint32_t(uint8_t(q0_hi_c.x)) << 16) | (uint32_t(uint8_t(q0_hi_c.y)) << 24);
+ q1 = uint32_t(uint8_t(q1_lo_c.x)) | (uint32_t(uint8_t(q1_lo_c.y)) << 8) |
+ (uint32_t(uint8_t(q1_hi_c.x)) << 16) | (uint32_t(uint8_t(q1_hi_c.y)) << 24);
+#else
+ const float inv_scale = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f;
+#pragma unroll
+ for (int k = 0; k < QK_NVFP4_SUB / 4; ++k) {
+ q0 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 0] * inv_col_scale, inv_scale)) << (8 * k);
+ q0 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 8] * inv_col_scale, inv_scale)) << (8 * k + 4);
+ q1 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 4] * inv_col_scale, inv_scale)) << (8 * k);
+ q1 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 12] * inv_col_scale, inv_scale)) << (8 * k + 4);
+ }
+#endif // CUDART_VERSION >= 12080
+
+ if constexpr (scatter) {
+#pragma unroll
+ for (int slot = 0; slot < n_expert_used; ++slot) {
+ const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];
+ block_fp4_mmq * yb = y + (k_block * ne1 + i);
+ uint32_t * yqs = reinterpret_cast<uint32_t *>(yb->qs);
+ yqs[2 * sub + 0] = q0;
+ yqs[2 * sub + 1] = q1;
+ reinterpret_cast<uint8_t *>(yb->d4)[sub] = fp8_code;
+ }
+ } else {
+ const int64_t blocks_per_col = (ne0 + QK_FP4_MMQ - 1) / QK_FP4_MMQ;
+ block_fp4_mmq * yb = y + (blockIdx.y * (blocks_per_col * ne1) + k_block * ne1 + blockIdx.x);
+ uint32_t * yqs = reinterpret_cast<uint32_t *>(yb->qs);
+ yqs[2 * sub + 0] = q0;
+ yqs[2 * sub + 1] = q1;
+ reinterpret_cast<uint8_t *>(yb->d4)[sub] = fp8_code;
+ }
+ }
+#else
+ GGML_UNUSED_VARS(x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, n_expert_used);
+ NO_DEVICE_CODE; // This is for Blackwell NVFP4 activations only.
+#endif // defined(BLACKWELL_MMA_AVAILABLE)
+
+}
+
+// quantize values in the format mxfp4 is stored which is interleaved nibbles
+// i.e. a block a0-a31 is represented as a0a16,a1a17 ...a15a31
+// scatter: grid over tokens, quantize once, write to all the token's compact rows
+template <bool scatter>
+static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x,
+ const int32_t * __restrict__ ids,
+ void * __restrict__ vy,
+ const int64_t ne00,
+ const int64_t s01,
+ const int64_t s02,
+ const int64_t s03,
+ const int64_t ne0,
+ const int ne1,
+ const int ne2,
+ const int n_expert_used) {
+ constexpr int vals_per_scale = 32;
+ constexpr int vals_per_warp = 2 * vals_per_scale; // Each warp processes 2 blocks of 32 = 64 values
+
+ const int warp_id = threadIdx.y;
+ const int lane_id_32 = threadIdx.x;
+
+ const int nwarps = blockDim.y;
+
+ const int64_t warp_start_offset = (blockIdx.y * nwarps + warp_id) * vals_per_warp;
+
+ if (warp_start_offset >= ne0) {
+ return;
+ }
+
+ const int64_t block_fp4_mmq_size = QK_FP4_MMQ;
+ const int64_t k_block = warp_start_offset / block_fp4_mmq_size;
+ const int64_t quad_idx_in_block = (warp_start_offset % block_fp4_mmq_size) / vals_per_warp;
+
+ const int group_id = lane_id_32 / 4;
+ const int lane_in_group = lane_id_32 % 4;
+ const int base = group_id * 2;
+
+ ggml_cuda_pdl_sync();
+ int64_t base_pos;
+ if constexpr (scatter) {
+ base_pos = (int64_t) blockIdx.x * s02; // one physical row per token
+ } else {
+ const int64_t i2 = blockIdx.z % ne2;
+ const int64_t i3 = blockIdx.z / ne2;
+ const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x;
+ base_pos = i3 * s03 + i2 * s02 + i01 * s01;
+ }
+
+ uint8_t scales[2];
+ char2 packed[2];
+
+#pragma unroll
+ for (int b = 0; b < 2; ++b) {
+ const int64_t i0 = warp_start_offset + b * vals_per_scale + lane_id_32;
+ const float xi = (i0 < ne00) ? x[base_pos + i0] : 0.0f;
+
+ float amax = fabsf(xi);
+#pragma unroll
+ for (int mask = 16; mask > 0; mask >>= 1) {
+ amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, WARP_SIZE));
+ }
+
+ const uint8_t e = compute_e8m0_scale(amax);
+ scales[b] = e;
+ const float inv_s = (amax == 0.0f) ? 0.0f : __frcp_rn(ggml_cuda_e8m0_to_fp32(e));
+
+#if CUDART_VERSION >= 12080
+ const float scaled_val = xi * inv_s;
+
+ const float val0 = __shfl_sync(0xFFFFFFFF, scaled_val, base, WARP_SIZE);
+ const float val1 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 16, WARP_SIZE);
+ const float val2 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 1, WARP_SIZE);
+ const float val3 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 17, WARP_SIZE);
+
+ __nv_fp4x4_e2m1 fp4_packed(make_float4(val0, val1, val2, val3));
+ packed[b] = *(char2 *) &fp4_packed;
+#else
+ // Fallback: manual FP4 conversion using LUT
+ const uint8_t q_val = ggml_cuda_float_to_fp4_e2m1(xi, inv_s);
+
+ const uint8_t q_lo_0 = __shfl_sync(0xFFFFFFFF, q_val, base, WARP_SIZE);
+ const uint8_t q_lo_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 1, WARP_SIZE);
+ const uint8_t q_hi_0 = __shfl_sync(0xFFFFFFFF, q_val, base + 16, WARP_SIZE);
+ const uint8_t q_hi_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 17, WARP_SIZE);
+
+ char2 q;
+ q.x = (q_hi_0 << 4) | q_lo_0;
+ q.y = (q_hi_1 << 4) | q_lo_1;
+ packed[b] = q;
+#endif // CUDART_VERSION >= 12080
+ }
+
+ block_fp4_mmq * y = (block_fp4_mmq *) vy;
+ if constexpr (scatter) {
+#pragma unroll
+ for (int slot = 0; slot < n_expert_used; ++slot) {
+ const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];
+ block_fp4_mmq * yb = y + (k_block * ne1 + i);
+ char2 * yqs2 = (char2 *) yb->qs;
+ if (lane_in_group == 0) {
+ yqs2[quad_idx_in_block * 16 + 0 * 8 + group_id] = packed[0];
+ yqs2[quad_idx_in_block * 16 + 1 * 8 + group_id] = packed[1];
+ }
+ if (lane_id_32 == 0) {
+ yb->d4[quad_idx_in_block] = (scales[1] << 8) | scales[0];
+ }
+ }
+ } else {
+ const int64_t ib0 = blockIdx.z * ((int64_t) ne1 * (ne0 / block_fp4_mmq_size));
+ block_fp4_mmq * yb = y + (ib0 + k_block * ne1 + blockIdx.x);
+ char2 * yqs2 = (char2 *) yb->qs;
+ if (lane_in_group == 0) {
+ yqs2[quad_idx_in_block * 16 + 0 * 8 + group_id] = packed[0];
+ yqs2[quad_idx_in_block * 16 + 1 * 8 + group_id] = packed[1];
+ }
+ if (lane_id_32 == 0) {
+ yb->d4[quad_idx_in_block] = (scales[1] << 8) | scales[0];
+ }
+ }
+ GGML_UNUSED(n_expert_used);
+}
+
+// FreeToken: 4 consecutive activations as float, from fp32, bf16 or fp16
+static __device__ __forceinline__ float4 load_float4(const float * x) {
+ return *(const float4 *) x;
+}
+
+static __device__ __forceinline__ float4 load_float4(const nv_bfloat16 * x) {
+ const uint2 v = *(const uint2 *) x;
+ const float2 lo = __bfloat1622float2(*(const nv_bfloat162 *) &v.x);
+ const float2 hi = __bfloat1622float2(*(const nv_bfloat162 *) &v.y);
+ return make_float4(lo.x, lo.y, hi.x, hi.y);
+}
+
+static __device__ __forceinline__ float4 load_float4(const half * x) {
+ const uint2 v = *(const uint2 *) x;
+ const float2 lo = __half22float2(*(const half2 *) &v.x);
+ const float2 hi = __half22float2(*(const half2 *) &v.y);
+ return make_float4(lo.x, lo.y, hi.x, hi.y);
+}
+
+// scatter: grid over tokens, quantize once, write to all the token's compact rows
+template <mmq_q8_1_ds_layout ds_layout, bool scatter, typename src_t>
+static __global__ void quantize_mmq_q8_1(
+ const src_t * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) {
+
+ constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32;
+ constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32;
+
+ const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;
+
+ if (i0 >= ne0) {
+ return;
+ }
+
+ const int64_t i00 = i0;
+ ggml_cuda_pdl_sync();
+
+ int64_t base_idx;
+ if constexpr (scatter) {
+ base_idx = (int64_t) blockIdx.x * s02; // one physical row per token
+ } else {
+ const int64_t i2 = blockIdx.z % ne2;
+ const int64_t i3 = blockIdx.z / ne2;
+ const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x;
+ base_idx = i3*s03 + i2*s02 + i01*s01;
+ }
+
+ block_q8_1_mmq * y = (block_q8_1_mmq *) vy;
+
+ const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel
+ const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block
+
+ // Load 4 floats per thread and calculate max. abs. value between them:
+ const float4 xi = i0 < ne00 ? load_float4(x + base_idx + i00) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ float amax = fabsf(xi.x);
+ amax = fmaxf(amax, fabsf(xi.y));
+ amax = fmaxf(amax, fabsf(xi.z));
+ amax = fmaxf(amax, fabsf(xi.w));
+
+ // Exchange max. abs. value between vals_per_scale/4 threads.
+#pragma unroll
+ for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) {
+ amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));
+ }
+
+ float sum;
+ if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
+ sum = xi.x + xi.y + xi.z + xi.w;
+
+ // Calculate sums across vals_per_sum/4 threads.
+#pragma unroll
+ for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) {
+ sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE);
+ }
+ }
+
+ const float d_inv = 127.0f / amax;
+ char4 q;
+ q.x = roundf(xi.x*d_inv);
+ q.y = roundf(xi.y*d_inv);
+ q.z = roundf(xi.z*d_inv);
+ q.w = roundf(xi.w*d_inv);
+ const float d = 1.0f / d_inv;
+
+ // write the block once (normal) or to each of the token's compact rows (scatter)
+ const int nwrite = scatter ? n_expert_used : 1;
+#pragma unroll
+ for (int slot = 0; slot < nwrite; ++slot) {
+ int64_t ib;
+ if constexpr (scatter) {
+ const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];
+ ib = k_block*ne1 + i;
+ } else {
+ const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel
+ ib = ib0 + k_block*ne1 + blockIdx.x;
+ }
+
+ // Write back 4 int8 values as a single 32 bit value for better memory bandwidth:
+ char4 * yqs4 = (char4 *) y[ib].qs;
+ yqs4[iqs/4] = q;
+
+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) {
+ if (iqs % 16 == 0 && iqs < 96) {
+ y[ib].d2s6[2 + iqs/16] = sum;
+ if (iqs % 64 == 0) {
+ y[ib].d2s6[iqs/64] = d;
+ }
+ }
+ } else if (iqs % 32 == 0) {
+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) {
+ y[ib].ds4[iqs/32] = make_half2(d, sum);
+ } else {
+ y[ib].d4[iqs/32] = d;
+ }
+ }
+ }
+ GGML_UNUSED(n_expert_used);
+}
+
+template <typename src_t>
+void quantize_row_q8_1_cuda(
+ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {
+ GGML_ASSERT(!ids);
+ GGML_ASSERT(ne0 % QK8_1 == 0);
+
+ const uint3 ne2_fastdiv = init_fastdiv_values(ne2);
+
+ const int64_t block_num_x = (ne0 + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
+ const dim3 num_blocks(block_num_x, ne1, ne2*ne3);
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);
+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream);
+ ggml_cuda_kernel_launch(quantize_q8_1<src_t>, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);
+ GGML_UNUSED(type_src0);
+}
+
+template <typename src_t>
+void quantize_mmq_q8_1_cuda(
+ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {
+ GGML_ASSERT(ne00 % 4 == 0);
+ GGML_ASSERT(ne0 % QK8_1_MMQ == 0);
+
+ // ne1 tends to assume the highest values, therefore use it as the "x" dimension of the CUDA grid:
+ const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
+ const dim3 num_blocks(ne1, block_num_y, ne2*ne3);
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+ switch (mmq_get_q8_1_ds_layout(type_src0)) {
+ case MMQ_Q8_1_DS_LAYOUT_D4:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, false, src_t>
+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_DS4:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, false, src_t>
+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_D2S6:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, false, src_t>
+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+// scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)
+template <typename src_t>
+void quantize_scatter_mmq_q8_1_cuda(
+ const src_t * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
+ const int64_t ne00, const int64_t stride_token, const int64_t ne0,
+ const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) {
+ GGML_ASSERT(ne00 % 4 == 0);
+ GGML_ASSERT(ne0 % QK8_1_MMQ == 0);
+
+ const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
+ const dim3 num_blocks(n_tokens, block_num_y, 1);
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+ switch (mmq_get_q8_1_ds_layout(type_src0)) {
+ case MMQ_Q8_1_DS_LAYOUT_D4:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, true, src_t><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_DS4:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, true, src_t><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
+ break;
+ case MMQ_Q8_1_DS_LAYOUT_D2S6:
+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, true, src_t><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
+ break;
+ default:
+ GGML_ABORT("fatal error");
+ }
+}
+
+// scatter=true reuses the quant kernels: grid over tokens, ids = inverse map (token slot -> compact row)
+void quantize_scatter_mmq_fp4_cuda(
+ const float * x, const int32_t * ids_src1_inv, void * vy, float * scale, const ggml_type type_src0, const bool use_aligned_float8,
+ const int64_t ne00, const int64_t stride_token, const int64_t ne0,
+ const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) {
+ GGML_ASSERT(ne0 > 0);
+ if (type_src0 == GGML_TYPE_NVFP4) {
+ GGML_ASSERT(scale);
+ GGML_ASSERT(ne00 % QK_NVFP4 == 0);
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+ const dim3 num_blocks(n_tokens, 1, 1);
+ if (use_aligned_float8) {
+ quantize_mmq_nvfp4<true, true><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, scale, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/nrows_dst, /*ne2=*/1, n_expert_used);
+ } else {
+ quantize_mmq_nvfp4<true, false><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, scale, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/nrows_dst, /*ne2=*/1, n_expert_used);
+ }
+ } else {
+ GGML_ASSERT(type_src0 == GGML_TYPE_MXFP4);
+ constexpr int nwarps = 8;
+ constexpr int vals_per_block = nwarps * 2 * QK_MXFP4;
+ const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block;
+ const dim3 block_size(WARP_SIZE, nwarps, 1);
+ const dim3 num_blocks(n_tokens, block_num_y, 1);
+ quantize_mmq_mxfp4<true><<<num_blocks, block_size, 0, stream>>>(
+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);
+ }
+}
+
+void quantize_mmq_fp4_cuda(
+ const float * x, const int32_t * ids, void * vy, float * scale, const ggml_type type_src0, const bool use_aligned_float8,
+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {
+ GGML_ASSERT(type_src0 == GGML_TYPE_MXFP4 || type_src0 == GGML_TYPE_NVFP4);
+ GGML_ASSERT(ne0 > 0);
+
+ if (type_src0 == GGML_TYPE_NVFP4) {
+ GGML_ASSERT(scale);
+ GGML_ASSERT(ne00 % QK_NVFP4 == 0);
+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+ const dim3 num_blocks(ne1, ne2 * ne3, 1);
+ if (use_aligned_float8) {
+ quantize_mmq_nvfp4<false, true><<<num_blocks, block_size, 0, stream>>>(
+ x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ } else {
+ quantize_mmq_nvfp4<false, false><<<num_blocks, block_size, 0, stream>>>(
+ x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ }
+ } else {
+ GGML_ASSERT(ne0 % (2 * QK_MXFP4) == 0);
+
+ constexpr int nwarps = 8;
+ constexpr int vals_per_warp = 2 * QK_MXFP4;
+ constexpr int vals_per_block = nwarps * vals_per_warp;
+
+ const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block;
+ const dim3 num_blocks(ne1, block_num_y, ne2 * ne3);
+ const dim3 block_size(WARP_SIZE, nwarps, 1);
+
+ quantize_mmq_mxfp4<false><<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);
+ }
+}
+
+#define FREETOKEN_QUANTIZE_INSTANCES(src_t) \
+ template void quantize_row_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \
+ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \
+ template void quantize_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \
+ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \
+ template void quantize_scatter_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, \
+ int64_t, int64_t, int64_t, int64_t, int, cudaStream_t);
+
+FREETOKEN_QUANTIZE_INSTANCES(float)
+FREETOKEN_QUANTIZE_INSTANCES(nv_bfloat16)
+FREETOKEN_QUANTIZE_INSTANCES(half)
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cuh
new file mode 100644
index 00000000..99283ea6
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cuh
@@ -0,0 +1,74 @@
+#pragma once
+
+#include "common.cuh"
+#include "mmq.cuh"
+
+#include <cstdint>
+
+#define CUDA_QUANTIZE_BLOCK_SIZE 256
+#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128
+
+static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access.");
+static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access.");
+
+typedef void (*quantize_cuda_t)(
+ const float * x, const int32_t * ids, void * vy,
+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
+
+// FreeToken: src_t is float, nv_bfloat16 or half
+template <typename src_t>
+void quantize_row_q8_1_cuda(
+ const src_t * x, const int32_t * ids, void * vy,
+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
+
+template <typename src_t>
+void quantize_mmq_q8_1_cuda(
+ const src_t * x, const int32_t * ids, void * vy,
+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
+
+void quantize_mmq_fp4_cuda(const float * x,
+ const int32_t * ids,
+ void * vy,
+ float * scale,
+ ggml_type type_src0,
+ bool use_aligned_float8,
+ int64_t ne00,
+ int64_t s01,
+ int64_t s02,
+ int64_t s03,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3,
+ cudaStream_t stream);
+
+// quantize each token once and scatter the block to its compact rows (via the inverse map)
+void quantize_scatter_mmq_fp4_cuda(const float * x,
+ const int32_t * ids_src1_inv,
+ void * vy,
+ float * scale,
+ ggml_type type_src0,
+ bool use_aligned_float8,
+ int64_t ne00,
+ int64_t stride_token,
+ int64_t ne0,
+ int64_t n_tokens,
+ int64_t nrows_dst,
+ int n_expert_used,
+ cudaStream_t stream);
+
+template <typename src_t>
+void quantize_scatter_mmq_q8_1_cuda(const src_t * x,
+ const int32_t * ids_src1_inv,
+ void * vy,
+ ggml_type type_src0,
+ int64_t ne00,
+ int64_t stride_token,
+ int64_t ne0,
+ int64_t n_tokens,
+ int64_t nrows_dst,
+ int n_expert_used,
+ cudaStream_t stream);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_s.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_s.cu
new file mode 100644
index 00000000..84ec8502
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_s.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_s.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_s.cu
new file mode 100644
index 00000000..583c4e5a
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_s.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu
new file mode 100644
index 00000000..edaf1560
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu
new file mode 100644
index 00000000..233d9342
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_s.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_s.cu
new file mode 100644
index 00000000..6092dc71
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_s.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu
new file mode 100644
index 00000000..1d5bd201
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu
new file mode 100644
index 00000000..eb02fab0
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu
new file mode 100644
index 00000000..1eb3b743
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-mxfp4.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-mxfp4.cu
new file mode 100644
index 00000000..0c1c7f29
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-mxfp4.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_MXFP4);
+
+DECL_MMQ_CASE_W4A4(GGML_TYPE_MXFP4);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-nvfp4.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-nvfp4.cu
new file mode 100644
index 00000000..bed8c8e2
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-nvfp4.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_NVFP4);
+
+DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q1_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q1_0.cu
new file mode 100644
index 00000000..f0686b0d
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q1_0.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q1_0);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_0.cu
new file mode 100644
index 00000000..750180e3
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_0.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q2_0);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_k.cu
new file mode 100644
index 00000000..6415369d
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_k.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q2_K);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q3_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q3_k.cu
new file mode 100644
index 00000000..ffb6213a
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q3_k.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q3_K);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_0.cu
new file mode 100644
index 00000000..0c0b0c8a
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_0.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q4_0);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_1.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_1.cu
new file mode 100644
index 00000000..ee67f694
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_1.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q4_1);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_k.cu
new file mode 100644
index 00000000..9eeb3cd7
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_k.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q4_K);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_0.cu
new file mode 100644
index 00000000..cc57fb97
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_0.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q5_0);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_1.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_1.cu
new file mode 100644
index 00000000..721ac790
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_1.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q5_1);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_k.cu
new file mode 100644
index 00000000..a2e90ffd
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_k.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q5_K);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q6_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q6_k.cu
new file mode 100644
index 00000000..470938fe
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q6_k.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q6_K);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q8_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q8_0.cu
new file mode 100644
index 00000000..974477bb
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q8_0.cu
@@ -0,0 +1,5 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../mmq.cuh"
+
+DECL_MMQ_CASE(GGML_TYPE_Q8_0);
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/unary.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/unary.cuh
new file mode 100644
index 00000000..04f3af64
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/unary.cuh
@@ -0,0 +1,123 @@
+#pragma once
+#include "common.cuh"
+
+#define CUDA_NEG_BLOCK_SIZE 256
+#define CUDA_STEP_BLOCK_SIZE 256
+#define CUDA_GELU_BLOCK_SIZE 256
+#define CUDA_SILU_BLOCK_SIZE 256
+#define CUDA_SILU_BACK_BLOCK_SIZE 256
+#define CUDA_TANH_BLOCK_SIZE 256
+#define CUDA_RELU_BLOCK_SIZE 256
+#define CUDA_SIGMOID_BLOCK_SIZE 256
+#define CUDA_HARDSIGMOID_BLOCK_SIZE 256
+#define CUDA_EXP_BLOCK_SIZE 256
+#define CUDA_HARDSWISH_BLOCK_SIZE 256
+#define CUDA_SQR_BLOCK_SIZE 256
+#define CUDA_SQRT_BLOCK_SIZE 256
+#define CUDA_SIN_BLOCK_SIZE 256
+#define CUDA_COS_BLOCK_SIZE 256
+#define CUDA_GLU_BLOCK_SIZE 256
+#define CUDA_XIELU_BLOCK_SIZE 256
+
+void ggml_cuda_op_abs(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sgn(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_neg(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_step(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_silu_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_gelu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_exp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sqrt(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_sin(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_cos(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_log(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_expm1(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_softplus(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_elu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_floor(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_ceil(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_round(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_trunc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_reglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_geglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_swiglu_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_geglu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node);
+
+void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_node, ggml_tensor * sqr_node);
+
+__device__ __forceinline__ float ggml_cuda_op_silu_single(float x) {
+ return x / (1.0f + expf(-x));
+}
+
+__device__ __forceinline__ float ggml_cuda_op_gelu_single(float x) {
+ const float GELU_COEF_A = 0.044715f;
+ const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
+
+ return 0.5f * x * (1.0f + tanhf(SQRT_2_OVER_PI * x * (1.0f + GELU_COEF_A * x * x)));
+}
+
+__device__ __forceinline__ float ggml_cuda_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) {
+ x = fminf(x, limit);
+ g = fmaxf(fminf(g, limit), -limit);
+
+ float out_glu = x / (1.0f + expf(-x * alpha));
+ out_glu = out_glu * (1.0f + g);
+ return out_glu;
+}
+
+__device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, float up, float limit) {
+ gate = fminf(gate, limit);
+ up = fmaxf(fminf(up, limit), -limit);
+
+ return ggml_cuda_op_silu_single(gate) * up;
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/vecdotq.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vecdotq.cuh
new file mode 100644
index 00000000..f65fe530
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vecdotq.cuh
@@ -0,0 +1,1380 @@
+#pragma once
+
+#include "common.cuh"
+
+#include <cstdint>
+
+static __device__ __forceinline__ int get_int_b1(const void * x, const int & i32) {
+ const uint8_t * x8 = (const uint8_t *) x;
+
+ int x32 = x8[4*i32 + 0] << 0;
+ x32 |= x8[4*i32 + 1] << 8;
+ x32 |= x8[4*i32 + 2] << 16;
+ x32 |= x8[4*i32 + 3] << 24;
+
+ return x32;
+}
+
+static __device__ __forceinline__ int get_int_b2(const void * x, const int & i32) {
+ const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment
+
+ int x32 = x16[2*i32 + 0] << 0;
+ x32 |= x16[2*i32 + 1] << 16;
+
+ return x32;
+}
+
+static __device__ __forceinline__ int get_int_b4(const void * x, const int & i32) {
+ return ((const int *) x)[i32]; // assume at least 4 byte alignment
+}
+
+// q4 contains 8 indices with 4 bit each.
+// This function selects those bytes from table that are at those indices and returns them as int2.
+// The first int contains the bytes with even indices in q4, the second int contains the bytes with odd indices in q4.
+static __device__ __forceinline__ int2 get_int_from_table_16(const int & q4, const int8_t * table) {
+#if defined(GGML_USE_HIP)
+ // Load the 16-byte table into four 32-bit unsigned integers.
+ const uint32_t *values = (const uint32_t *)table;
+
+ const uint32_t q_even = q4;
+ const uint32_t q_odd = (q4 >> 4);
+
+ // Perform lookups in the lower half of the table (indices 0-7).
+ uint32_t v_even_low = __builtin_amdgcn_perm(values[1], values[0], q_even & 0x07070707);
+ uint32_t v_odd_low = __builtin_amdgcn_perm(values[1], values[0], q_odd & 0x07070707);
+
+ // Perform lookups in the upper half of the table (indices 8-15).
+ uint32_t v_even_high = __builtin_amdgcn_perm(values[3], values[2], q_even & 0x07070707);
+ uint32_t v_odd_high = __builtin_amdgcn_perm(values[3], values[2], q_odd & 0x07070707);
+
+ // Select between the low and high results based on the MSB of each index nibble.
+ uint32_t mask_even = 0x03020100 | ((q_even & 0x08080808) >> 1);
+ uint32_t res_x = __builtin_amdgcn_perm(v_even_high, v_even_low, mask_even);
+ uint32_t mask_odd = 0x03020100 | ((q_odd & 0x08080808) >> 1);
+ uint32_t res_y = __builtin_amdgcn_perm(v_odd_high, v_odd_low, mask_odd);
+
+ return make_int2(res_x, res_y);
+#elif !defined(GGML_USE_MUSA)
+ // CUDA does not have an instruction for selecting bytes with 4 bit indices.
+ // However, __byte_perm is an instruction that selects bytes with 3 bit indices that can be used instead.
+ const uint32_t * table32 = (const uint32_t *) table;
+
+ // __byte_perm selects bytes based on the lower 16 bits in its third argument.
+ // Therefore, do 2 iterations over the 32 bits in q4 with 0 and 16 shift.
+ // To handle the fourth bit, first call _byte_perm both for the low and the high 64 bit of table, using the low 3 bits.
+ // Then, call __byte_perm again to select from the low and high bytes based on the fourth bit.
+ uint32_t tmp[2];
+ const uint32_t low_high_selection_indices = (0x32103210 | ((q4 & 0x88888888) >> 1));
+#pragma unroll
+ for (uint32_t i = 0; i < 2; ++i) {
+ const uint32_t shift = 16 * i;
+
+ const uint32_t low = __byte_perm(table32[0], table32[1], q4 >> shift);
+ const uint32_t high = __byte_perm(table32[2], table32[3], q4 >> shift);
+ tmp[i] = __byte_perm(low, high, low_high_selection_indices >> shift);
+ }
+
+ // tmp contains the bytes from tyble in the same order as the 4 bit indices in q4.
+ // However, for the result we need ints with all even/odd 4 bit indices in q4.
+ // Therefore, 2 more calls to __byte_perm to put the bytes in the correct order.
+ return make_int2(__byte_perm(tmp[0], tmp[1], 0x6420), __byte_perm(tmp[0], tmp[1], 0x7531));
+#else
+ // Generic implementation.
+ const int q0_32 = (q4 >> 0) & 0x0F0F0F0F;
+ const int8_t * q0_8 = (const int8_t *) &q0_32;
+ const char4 val0_8 = make_char4(
+ table[q0_8[0]], table[q0_8[1]], table[q0_8[2]], table[q0_8[3]]);
+
+ const int q1_32 = (q4 >> 4) & 0x0F0F0F0F;
+ const int8_t * q1_8 = (const int8_t *) &q1_32;
+ const char4 val1_8 = make_char4(
+ table[q1_8[0]], table[q1_8[1]], table[q1_8[2]], table[q1_8[3]]);
+
+ return make_int2(*((const int *) &val0_8), *((const int *) &val1_8));
+#endif
+}
+
+static __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) {
+ // v is a 7 bit int, with the 8th sign being encodable as popcnt
+ // with xor we can "correct" the bit instead of having to mask
+ const uint32_t p = __popc(v) & 1;
+ const uint32_t s = v ^ p << 7;
+ // broadcast over uint to allow for 0x08040201 / 0x80402010 as selectors
+ return s * 0x01010101;
+}
+
+// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called
+// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q
+
+#define VDR_Q1_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism
+#define VDR_Q1_0_Q8_1_MMQ 4 // Q1_0 has 128 bits (4 ints) per block
+
+#define VDR_Q2_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism
+#define VDR_Q2_0_Q8_1_MMQ 2 // Q2_0 group 64: 128 bits (4 ints) per block, 2 32-element chunks
+
+#define VDR_Q4_0_Q8_1_MMVQ 2
+#define VDR_Q4_0_Q8_1_MMQ 4
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
+ const int * v, const int * u, const float & d4, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
+
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);
+ }
+
+ const float2 ds8f = __half22float2(ds8);
+
+ // second part effectively subtracts 8 from each quant value
+ return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);
+}
+
+#define VDR_Q4_1_Q8_1_MMVQ 2
+#define VDR_Q4_1_Q8_1_MMQ 4
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
+ const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
+
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);
+ }
+
+#ifdef FAST_FP16_AVAILABLE
+ const float2 tmp = __half22float2(__hmul2(dm4, ds8));
+ const float d4d8 = tmp.x;
+ const float m4s8 = tmp.y;
+#else
+ const float2 dm4f = __half22float2(dm4);
+ const float2 ds8f = __half22float2(ds8);
+ const float d4d8 = dm4f.x * ds8f.x;
+ const float m4s8 = dm4f.y * ds8f.y;
+#endif // FAST_FP16_AVAILABLE
+
+ // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
+ return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
+}
+
+#define VDR_Q5_0_Q8_1_MMVQ 2
+#define VDR_Q5_0_Q8_1_MMQ 4
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
+ const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
+
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
+ }
+
+ const float2 ds8f = __half22float2(ds8);
+
+ // second part effectively subtracts 16 from each quant value
+ return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);
+}
+
+#define VDR_Q5_1_Q8_1_MMVQ 2
+#define VDR_Q5_1_Q8_1_MMQ 4
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
+ const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
+
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
+ }
+
+#ifdef FAST_FP16_AVAILABLE
+ const float2 tmp = __half22float2(__hmul2(dm5, ds8));
+ const float d5d8 = tmp.x;
+ const float m5s8 = tmp.y;
+#else
+ const float2 dm5f = __half22float2(dm5);
+ const float2 ds8f = __half22float2(ds8);
+ const float d5d8 = dm5f.x * ds8f.x;
+ const float m5s8 = dm5f.y * ds8f.y;
+#endif // FAST_FP16_AVAILABLE
+
+ // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
+ return sumi*d5d8 + m5s8 / (QI5_1 / vdr);
+}
+
+#define VDR_Q8_0_Q8_1_MMVQ 2
+#define VDR_Q8_0_Q8_1_MMQ 8
+
+template <typename T, int vdr> static __device__ __forceinline__ T vec_dot_q8_0_q8_1_impl(
+ const int * v, const int * u, const T & d8_0, const T & d8_1) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
+ }
+
+ return d8_0*d8_1 * ((T) sumi);
+}
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
+ const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = 0; i < vdr; ++i) {
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
+ }
+
+#ifdef FAST_FP16_AVAILABLE
+ const float2 tmp = __half22float2(__hmul2(dm8, ds8));
+ const float d8d8 = tmp.x;
+ const float m8s8 = tmp.y;
+#else
+ const float2 dm8f = __half22float2(dm8);
+ const float2 ds8f = __half22float2(ds8);
+ const float d8d8 = dm8f.x * ds8f.x;
+ const float m8s8 = dm8f.y * ds8f.y;
+#endif // FAST_FP16_AVAILABLE
+
+ // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
+ return sumi*d8d8 + m8s8 / (QI8_1 / vdr);
+}
+
+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_16_q8_1_impl(
+ const int * v, const int * u, const float * d8_0, const float & d8_1) {
+
+ float sumf = 0.0f;
+
+#pragma unroll
+ for (int i0 = 0; i0 < vdr; i0 += QI8_0/2) {
+ int sumi = 0;
+
+#pragma unroll
+ for (int i = i0; i < i0 + QI8_0/2; ++i) {
+ // SIMD dot product of quantized values
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
+ }
+
+ sumf += d8_0[i0/(QI8_0/2)]*sumi;
+ }
+
+ return d8_1*sumf;
+}
+
+#define VDR_MXFP4_Q8_1_MMVQ 2
+#define VDR_MXFP4_Q8_1_MMQ 4
+
+static __device__ __forceinline__ float vec_dot_mxfp4_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_mxfp4 * bq4 = (const block_mxfp4 *) vbq + kbx;
+
+ const int * q8 = (const int *) bq8_1->qs + iqs;
+
+ int sumi = 0;
+#pragma unroll
+ for (int l = 0; l < VDR_MXFP4_Q8_1_MMVQ; ++l) {
+ const int aux_q4 = get_int_b1(bq4->qs, iqs + l);
+ const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);
+
+ sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);
+ sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);
+ }
+
+ const float d = ggml_cuda_e8m0_to_fp32(bq4->e) * 0.5f * __low2float(bq8_1->ds);
+ return d * sumi;
+}
+
+#define VDR_NVFP4_Q8_1_MMVQ 4
+#define VDR_NVFP4_Q8_1_MMQ 8
+
+static __device__ __forceinline__ float vec_dot_nvfp4_q8_1(
+ const void * __restrict__ vbq,
+ const block_q8_1 * __restrict__ bq8_1,
+ const int32_t & kbx,
+ const int32_t & iqs) {
+
+ const block_nvfp4 * bq4 = (const block_nvfp4 *) vbq + kbx;
+ float sum = 0.0f;
+#pragma unroll
+ for (int i = 0; i < VDR_NVFP4_Q8_1_MMVQ/2; i++) {
+ const int32_t iqs0 = iqs + 2*i;
+ const int32_t iqs1 = iqs0 + 1;
+ const int32_t is = iqs0 >> 1;
+ const int2 v0 = get_int_from_table_16(get_int_b4(bq4->qs, iqs0), kvalues_mxfp4);
+ const int2 v1 = get_int_from_table_16(get_int_b4(bq4->qs, iqs1), kvalues_mxfp4);
+ const block_q8_1 * bq8 = bq8_1 + (is >> 1);
+ const int32_t i8 = ((is & 1) << 2);
+
+ int sumi = ggml_cuda_dp4a(v0.x, get_int_b4(bq8->qs, i8 + 0), 0);
+ sumi = ggml_cuda_dp4a(v0.y, get_int_b4(bq8->qs, i8 + 2), sumi);
+ sumi = ggml_cuda_dp4a(v1.x, get_int_b4(bq8->qs, i8 + 1), sumi);
+ sumi = ggml_cuda_dp4a(v1.y, get_int_b4(bq8->qs, i8 + 3), sumi);
+
+ const float d = ggml_cuda_ue4m3_to_fp32(bq4->d[is]) * __low2float(bq8->ds);
+ sum += d * float(sumi);
+ }
+
+ return sum;
+}
+#define VDR_Q2_K_Q8_1_MMVQ 1
+#define VDR_Q2_K_Q8_1_MMQ 4
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
+ const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
+ const half2 & dm2, const float * __restrict__ d8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR2_K; ++i) {
+ const int sc = scales[2*i];
+
+ const int vi = (v >> (2*i)) & 0x03030303;
+
+ sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
+
+ // fill int with 4x m
+ int m = sc >> 4;
+ m |= m << 8;
+ m |= m << 16;
+ sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
+ }
+
+ const float2 dm2f = __half22float2(dm2);
+
+ return dm2f.x*sumf_d - dm2f.y*sumf_m;
+}
+
+// contiguous v/x + u/y values
+template <int ns8>
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const half2 * dm2, const float & d8, const half2 * s8) {
+
+ float sumf = 0.0f;
+ float sumf_d8 = 0.0f;
+
+#pragma unroll
+ for (int i0 = 0; i0 < QR2_K*VDR_Q2_K_Q8_1_MMQ; i0 += QI8_1) {
+ const float2 dm2f0 = __half22float2(dm2[i0/(QI8_1/2) + 0]);
+ int sumi_d0 = 0;
+
+ const float2 dm2f1 = __half22float2(dm2[i0/(QI8_1/2) + 1]);
+ int sumi_d1 = 0;
+
+#pragma unroll
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
+ sumi_d0 = ggml_cuda_dp4a(v[i], u[i], sumi_d0);
+ }
+ sumf_d8 += dm2f0.x * sumi_d0;
+
+#pragma unroll
+ for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {
+ sumi_d1 = ggml_cuda_dp4a(v[i], u[i], sumi_d1);
+ }
+ sumf_d8 += dm2f1.x * sumi_d1;
+
+ if (i0/QI8_1 < ns8) {
+ const float2 s8f = __half22float2(s8[i0/QI8_1]);
+ sumf -= dm2f0.y*s8f.x;
+ sumf -= dm2f1.y*s8f.y;
+ } else {
+ int sumi_m0 = 0;
+#pragma unroll
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
+ sumi_m0 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m0);
+ }
+ sumf_d8 -= dm2f0.y * sumi_m0;
+
+ int sumi_m1 = 0;
+#pragma unroll
+ for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {
+ sumi_m1 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m1);
+ }
+ sumf_d8 -= dm2f1.y * sumi_m1;
+ }
+ }
+
+ return sumf + d8*sumf_d8;
+}
+
+#define VDR_Q3_K_Q8_1_MMVQ 1
+#define VDR_Q3_K_Q8_1_MMQ 2
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
+ const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
+ const int & scale_offset, const float & d3, const float * __restrict__ d8) {
+
+ float sumf = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR3_K; ++i) {
+ const int isc = scale_offset + 2*i;
+
+ const int isc_low = isc % (QK_K/32);
+ const int sc_shift_low = 4 * (isc / (QK_K/32));
+ const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
+
+ const int isc_high = isc % (QK_K/64);
+ const int sc_shift_high = 2 * (isc / (QK_K/64));
+ const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
+
+ const int sc = (sc_low | sc_high) - 32;
+
+ const int vil = (vl >> (2*i)) & 0x03030303;
+
+ const int vih = ((vh >> i) << 2) & 0x04040404;
+
+ const int vi = __vsub4(vil, vih);
+
+ sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
+ }
+
+ return d3 * sumf;
+}
+
+// contiguous v/x + u/y values
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
+ const float & d3, const float & d8) {
+
+ int sumi = 0;
+
+#pragma unroll
+ for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {
+ int sumi_sc = 0;
+
+#pragma unroll
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
+ sumi_sc = ggml_cuda_dp4a(v[i], u[i], sumi_sc); // SIMD dot product
+ }
+
+ sumi += sumi_sc * scales[i0 / (QI8_1/2)];
+ }
+
+ return d3*d8 * sumi;
+}
+
+#define VDR_Q4_K_Q8_1_MMVQ 2
+#define VDR_Q4_K_Q8_1_MMQ 8
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
+ const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR4_K; ++i) {
+ const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
+ const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
+
+ const int dot1 = ggml_cuda_dp4a(v1i, u[2*i+1], ggml_cuda_dp4a(v0i, u[2*i+0], 0)); // SIMD dot product
+ const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+1], ggml_cuda_dp4a(0x01010101, u[2*i+0], 0)); // sum of u
+
+ sumf_d += d8[i] * (dot1 * sc[i]);
+ sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
+ }
+
+ const float2 dm4f = __half22float2(dm4);
+
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
+}
+
+// contiguous v/x + u/y values
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {
+ int sumi_d = 0;
+
+#pragma unroll
+ for (int j = 0; j < QI8_1; ++j) {
+ sumi_d = ggml_cuda_dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product
+ }
+
+ const float2 ds8f = __half22float2(ds8[i]);
+
+ sumf_d += ds8f.x * (sc[i] * sumi_d);
+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
+ }
+
+ const float2 dm4f = __half22float2(dm4);
+
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
+}
+
+#define VDR_Q5_K_Q8_1_MMVQ 2
+#define VDR_Q5_K_Q8_1_MMQ 8
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
+ const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
+ const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR5_K; ++i) {
+ const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;
+ const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;
+
+ const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
+ const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
+
+ const int v0i = vl0i | vh0i;
+ const int v1i = vl1i | vh1i;
+
+ const int dot1 = ggml_cuda_dp4a(v0i, u[2*i+0], ggml_cuda_dp4a(v1i, u[2*i+1], 0)); // SIMD dot product
+ const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+0], ggml_cuda_dp4a(0x01010101, u[2*i+1], 0)); // sum of u
+
+ sumf_d += d8[i] * (dot1 * sc[i]);
+ sumf_m += d8[i] * (dot2 * m[i]);
+
+ }
+
+ const float2 dm5f = __half22float2(dm5);
+
+ return dm5f.x*sumf_d - dm5f.y*sumf_m;
+}
+
+// contiguous v/x + u/y values
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
+
+ float sumf_d = 0.0f;
+ float sumf_m = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {
+ int sumi_d = 0;
+
+#pragma unroll
+ for (int j = 0; j < QI8_1; ++j) {
+ sumi_d = ggml_cuda_dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product
+ }
+
+ const float2 ds8f = __half22float2(ds8[i]);
+
+ sumf_d += ds8f.x * (sc[i] * sumi_d);
+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
+ }
+
+ const float2 dm4f = __half22float2(dm4);
+
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
+}
+
+#define VDR_Q6_K_Q8_1_MMVQ 1
+#define VDR_Q6_K_Q8_1_MMQ 8
+
+// contiguous v/x values
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
+ const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
+ const float & d, const float * __restrict__ d8) {
+
+ float sumf = 0.0f;
+
+#pragma unroll
+ for (int i = 0; i < QR6_K; ++i) {
+ const int sc = scales[4*i];
+
+ const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
+
+ const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
+
+ const int vi = __vsub4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
+
+ sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
+ }
+
+ return d*sumf;
+}
+
+// contiguous v/x + u/y values
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
+ const float & d6, const float * __restrict__ d8) {
+
+ float sumf_d = 0.0f;
+
+ const int sc_packed = get_int_b4(sc, 0);
+ const int8_t * sc_reg = (const int8_t *) &sc_packed;
+
+#pragma unroll
+ for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {
+ int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale
+
+#pragma unroll
+ for (int i = i0; i < i0 + 2; ++i) {
+ sumi_d.x = ggml_cuda_dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product
+ sumi_d.x = ggml_cuda_dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product
+
+ sumi_d.y = ggml_cuda_dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product
+ sumi_d.y = ggml_cuda_dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product
+ }
+
+ sumf_d += d8[i0/4] * (sc_reg[i0/2+0]*sumi_d.x + sc_reg[i0/2+1]*sumi_d.y);
+ }
+
+ return d6 * sumf_d;
+}
+
+static __device__ __forceinline__ float vec_dot_q1_0_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q1_0 * bq1_0 = (const block_q1_0 *) vbq + kbx;
+
+ // Q1_0: 128 elements with ONE scale
+ // Q8_1: 32 elements per block with individual scales
+ // iqs selects which of the 4 chunks of 32 elements to process (0-3)
+
+ const float d1 = bq1_0->d;
+ const int16_t * qs = (const int16_t *) bq1_0->qs + iqs * 2;
+
+ // Process only the chunk specified by iqs
+ const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;
+
+ int sumi = 0;
+#pragma unroll
+ for (int j = 0; j < 2; ++j) {
+ const int q = qs[j];
+
+ const int u0 = get_int_b4(bq8_1_chunk->qs, j*4+0);
+ const int u1 = get_int_b4(bq8_1_chunk->qs, j*4+1);
+ const int u2 = get_int_b4(bq8_1_chunk->qs, j*4+2);
+ const int u3 = get_int_b4(bq8_1_chunk->qs, j*4+3);
+
+ // unpack crumbs into nibble indices
+ const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13]
+ const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15]
+ // unpack nibbles into byte values
+ const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0);
+ const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0);
+ const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16);
+ const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16);
+ // unshuffle values
+ const int v0 = __byte_perm(s0, s1, 0x5410);
+ const int v1 = __byte_perm(s0, s1, 0x7632);
+ const int v2 = __byte_perm(s2, s3, 0x5410);
+ const int v3 = __byte_perm(s2, s3, 0x7632);
+
+ sumi = ggml_cuda_dp4a(v0, u0, sumi);
+ sumi = ggml_cuda_dp4a(v1, u1, sumi);
+ sumi = ggml_cuda_dp4a(v2, u2, sumi);
+ sumi = ggml_cuda_dp4a(v3, u3, sumi);
+ }
+
+ // Apply Q1_0's single scale and this chunk's Q8_1 scale
+ const float d8 = __low2float(bq8_1_chunk->ds);
+ return d1 * d8 * sumi;
+}
+
+static __device__ __forceinline__ float vec_dot_q2_0_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx;
+
+ // Q2_0 (group 64): 64 elements with ONE scale, 2 bits per element (4 elements per byte)
+ // Q8_1: 32 elements per block with individual scales
+ // iqs selects which of the 2 chunks of 32 elements to process (0-1)
+
+ const float d2 = bq2_0->d;
+ const int16_t * qs = (const int16_t *) bq2_0->qs + iqs * 4;
+
+ // Process only the chunk specified by iqs
+ const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;
+
+ int sumi = 0;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int q = qs[j];
+ const int u = get_int_b4(bq8_1_chunk->qs, j*2+0);
+ const int v = get_int_b4(bq8_1_chunk->qs, j*2+1);
+
+#if defined(GGML_USE_HIP)
+ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);
+ const uint32_t qy_bits = q >> 8;
+ const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);
+ const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);
+ const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);
+#else
+ // unpack even and odd crumbs into byte values
+ const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);
+ const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);
+ // unshuffle values
+ const int qx = __byte_perm(qe, qo, 0x5140);
+ const int qy = __byte_perm(qe, qo, 0x7362);
+#endif // defined(GGML_USE_HIP)
+
+ sumi = ggml_cuda_dp4a(u, qx, sumi);
+ sumi = ggml_cuda_dp4a(v, qy, sumi);
+ }
+
+ // Apply Q2_0's single scale and this chunk's Q8_1 scale
+ const float d8 = __low2float(bq8_1_chunk->ds);
+ return d2 * d8 * sumi;
+}
+
+static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq + kbx;
+
+ int v[VDR_Q4_0_Q8_1_MMVQ];
+ int u[2*VDR_Q4_0_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
+ v[i] = get_int_b2(bq4_0->qs, iqs + i);
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_0);
+ }
+
+ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);
+}
+
+
+static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq + kbx;
+
+ int v[VDR_Q4_1_Q8_1_MMVQ];
+ int u[2*VDR_Q4_1_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
+ v[i] = get_int_b4(bq4_1->qs, iqs + i);
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_1);
+ }
+
+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
+}
+
+static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq + kbx;
+
+ int vl[VDR_Q5_0_Q8_1_MMVQ];
+ int vh[VDR_Q5_0_Q8_1_MMVQ];
+ int u[2*VDR_Q5_0_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
+ vl[i] = get_int_b2(bq5_0->qs, iqs + i);
+ vh[i] = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_0);
+ }
+
+ return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);
+}
+
+static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq + kbx;
+
+ int vl[VDR_Q5_1_Q8_1_MMVQ];
+ int vh[VDR_Q5_1_Q8_1_MMVQ];
+ int u[2*VDR_Q5_1_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
+ vl[i] = get_int_b4(bq5_1->qs, iqs + i);
+ vh[i] = get_int_b4(bq5_1->qh, 0) >> (4 * (iqs + i));
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_1);
+ }
+
+ return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
+}
+
+static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq + kbx;
+
+ int v[VDR_Q8_0_Q8_1_MMVQ];
+ int u[VDR_Q8_0_Q8_1_MMVQ];
+
+#pragma unroll
+ for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
+ v[i] = get_int_b2(bq8_0->qs, iqs + i);
+ u[i] = get_int_b4(bq8_1->qs, iqs + i);
+ }
+
+ return vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));
+}
+
+static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx;
+
+ const int bq8_offset = QR2_K * (iqs / QI8_1);
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
+
+ const uint8_t * scales = bq2_K->scales + scale_offset;
+
+ const int v = get_int_b4(bq2_K->qs, iqs);
+ int u[QR2_K];
+ float d8[QR2_K];
+
+#pragma unroll
+ for (int i = 0; i < QR2_K; ++ i) {
+ u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
+ }
+
+ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
+}
+
+static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx;
+
+ const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
+
+ const float d = bq3_K->d;
+
+ const int vl = get_int_b2(bq3_K->qs, iqs);
+
+ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
+ const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
+
+ int u[QR3_K];
+ float d8[QR3_K];
+
+#pragma unroll
+ for (int i = 0; i < QR3_K; ++i) {
+ u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
+ }
+
+ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
+}
+
+static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q4_K * bq4_K = (const block_q4_K *) vbq + kbx;
+
+ int v[2];
+ int u[2*QR4_K];
+ float d8[QR4_K];
+
+ // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
+ const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));
+
+ // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
+ // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
+ // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
+ // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108
+
+ const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
+ v[0] = q4[0];
+ v[1] = q4[4];
+
+ // branchless so nvcc can hoist this out of the ncols_dst loop
+ const uint16_t * scales = (const uint16_t *)bq4_K->scales;
+ const int j = bq8_offset/2;
+ const int jm = j & 1;
+
+ const uint32_t s0 = scales[jm + 0];
+ const uint32_t s2 = scales[jm + 2];
+ const uint32_t s4 = scales[jm + 4];
+
+ const uint32_t hi = (uint32_t) -(int32_t) (j >= 2);
+
+ uint16_t aux[2];
+ aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi));
+ aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi));
+ const uint8_t * sc = (const uint8_t *)aux;
+ const uint8_t * m = sc + 2;
+
+ for (int i = 0; i < QR4_K; ++i) {
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
+ d8[i] = __low2float(bq8i->ds);
+
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
+ u[2*i+0] = q8[0];
+ u[2*i+1] = q8[4];
+ }
+
+ return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
+}
+
+static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q5_K * bq5_K = (const block_q5_K *) vbq + kbx;
+
+ int vl[2];
+ int vh[2];
+ int u[2*QR5_K];
+ float d8[QR5_K];
+
+ const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));
+ const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
+ const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));
+
+ vl[0] = ql[0];
+ vl[1] = ql[4];
+
+ vh[0] = qh[0] >> bq8_offset;
+ vh[1] = qh[4] >> bq8_offset;
+
+ // same as q4_K
+ const uint16_t * scales = (const uint16_t *)bq5_K->scales;
+ const int j = bq8_offset/2;
+ const int jm = j & 1;
+
+ const uint32_t s0 = scales[jm + 0];
+ const uint32_t s2 = scales[jm + 2];
+ const uint32_t s4 = scales[jm + 4];
+
+ const uint32_t hi = (uint32_t) -(int32_t) (j >= 2);
+
+ uint16_t aux[2];
+ aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi));
+ aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi));
+
+ const uint8_t * sc = (const uint8_t *)aux;
+ const uint8_t * m = sc + 2;
+
+#pragma unroll
+ for (int i = 0; i < QR5_K; ++i) {
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
+ d8[i] = __low2float(bq8i->ds);
+
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
+ u[2*i+0] = q8[0];
+ u[2*i+1] = q8[4];
+ }
+
+ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
+}
+
+static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_q6_K * bq6_K = (const block_q6_K *) vbq + kbx;
+
+ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);
+ const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
+ const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
+
+ const int vl = get_int_b2(bq6_K->ql, iqs);
+ const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
+
+ const int8_t * scales = bq6_K->scales + scale_offset;
+
+ int u[QR6_K];
+ float d8[QR6_K];
+
+#pragma unroll
+ for (int i = 0; i < QR6_K; ++i) {
+ u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
+ d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
+ }
+
+ return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);
+}
+
+#define VDR_IQ2_XXS_Q8_1_MMVQ 2
+#define VDR_IQ2_XXS_Q8_1_MMQ 2
+
+static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq + kbx;
+
+ const int q2 = get_int_b2(bq2->qs, iqs);
+ const uint8_t * aux8 = (const uint8_t *) &q2;
+ const uint32_t aux32 = get_int_b2(bq2->qs, iqs + 1);
+
+ int sumi = 0;
+#pragma unroll
+ for (int k0 = 0; k0 < 8; k0 += 2) {
+ const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[k0/2]];
+ const uint32_t signs = unpack_ksigns(aux32 >> (7 * k0 / 2));
+
+ const int signs0 = __vcmpne4(signs & 0x08040201, 0);
+ const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0);
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, k0 + 0);
+ sumi = ggml_cuda_dp4a(grid0, u0, sumi);
+
+ const int signs1 = __vcmpne4(signs & 0x80402010, 0);
+ const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, k0 + 1);
+ sumi = ggml_cuda_dp4a(grid1, u1, sumi);
+ }
+
+ const int ls = aux32 >> 27 | 1; // (scale * 2 + 1)
+ sumi = sumi * ls / 8; // (sumi * scale + sumi / 2) / 4
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ2_XS_Q8_1_MMVQ 2
+#define VDR_IQ2_XS_Q8_1_MMQ 2
+
+static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq + kbx;
+
+ const int2 q2_packed = make_int2(get_int_b2(bq2->qs, iqs + 0), get_int_b2(bq2->qs, iqs + 1));
+ const uint16_t * q2 = (const uint16_t *) &q2_packed;
+ const int ls0 = bq2->scales[iqs/2] & 0x0F;
+ const int ls1 = bq2->scales[iqs/2] >> 4;
+
+ int sumi0 = 0;
+ int sumi1 = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l0/2] & 0x1FF];
+ const uint32_t signs = unpack_ksigns(q2[l0/2] >> 9);
+
+ const int signs0 = __vcmpne4(signs & 0x08040201, 0);
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+
+ const int signs1 = __vcmpne4(signs & 0x80402010, 0);
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+ if (l0 < 4) {
+ sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);
+ sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);
+ } else {
+ sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);
+ sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);
+ }
+ }
+ const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ2_S_Q8_1_MMVQ 2
+#define VDR_IQ2_S_Q8_1_MMQ 2
+
+static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq2_s * bq2 = (const block_iq2_s *) vbq + kbx;
+
+ const int qs_packed = get_int_b2(bq2->qs, iqs/2);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bq2->qh[iqs/2];
+
+ const int signs_packed_32 = get_int_b2(bq2->qs, QK_K/32 + iqs/2);
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+
+ const int ls0 = bq2->scales[iqs/2] & 0x0F;
+ const int ls1 = bq2->scales[iqs/2] >> 4;
+
+ int sumi0 = 0;
+ int sumi1 = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int * grid_pos = (const int *)(iq2s_grid + (qs[l0/2] | ((qh << (8-l0)) & 0x300)));
+
+ const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
+ const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
+
+ const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);
+ const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);
+
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+ if (l0 < 4) {
+ sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);
+ sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);
+ } else {
+ sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);
+ sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);
+ }
+ }
+ const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;
+
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ3_XXS_Q8_1_MMVQ 2
+#define VDR_IQ3_XXS_Q8_1_MMQ 2
+
+static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq3_xxs * bq3 = (const block_iq3_xxs *) vbq + kbx;
+
+ const int2 q3_packed = make_int2(get_int_b2(bq3->qs, iqs), get_int_b2(bq3->qs, iqs+1));
+ const uint8_t * q3 = (const uint8_t *) &q3_packed;
+ const uint32_t aux32 = get_int_b2(bq3->qs, QK_K/16 + iqs/2);
+
+ int sumi = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int2 grid_pos = make_int2(iq3xxs_grid[q3[l0 + 0]], iq3xxs_grid[q3[l0 + 1]]);
+ const uint32_t signs = unpack_ksigns(aux32 >> (7*l0/2));
+
+ const int signs0 = __vcmpne4(signs & 0x08040201, 0);
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+
+ const int signs1 = __vcmpne4(signs & 0x80402010, 0);
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+ sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
+ sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
+ }
+
+ const int ls = aux32 >> 28;
+ sumi = (ls*sumi + sumi/2)/2;
+ const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ3_S_Q8_1_MMVQ 2
+#define VDR_IQ3_S_Q8_1_MMQ 2
+
+// TODO: don't use lookup table for signs
+static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx;
+
+ const int2 qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1));
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bq3->qh[iqs/2];
+
+ const int signs_packed_32 = get_int_b2(bq3->signs, iqs/2);
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+
+ int sumi = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int2 grid_pos = make_int2(
+ iq3s_grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)],
+ iq3s_grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]);
+
+ const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
+ const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
+
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+ sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
+ sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
+ }
+
+ sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);
+
+ const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
+ return d * sumi;
+}
+
+#define VDR_IQ1_S_Q8_1_MMVQ 1
+#define VDR_IQ1_S_Q8_1_MMQ 1
+
+static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq + kbx;
+
+ const int qs_packed = get_int_b2(bq1->qs, iqs);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ const int qh = bq1->qh[iqs];
+
+ int sumi = 0;
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int grid = iq1s_grid_gpu[qs[l0/2] | (((qh >> 3*(l0/2)) & 0x07) << 8)];
+
+ const int grid0 = (grid >> 0) & 0x0F0F0F0F;
+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;
+
+ const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
+
+ sumi = ggml_cuda_dp4a(grid0, u0, sumi);
+ sumi = ggml_cuda_dp4a(grid1, u1, sumi);
+ }
+
+ const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);
+ const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);
+ const float2 ds = __half22float2(bq8_1[iqs].ds);
+ return d1q * (ds.x*sumi + ds.y*delta);
+}
+
+#define VDR_IQ1_M_Q8_1_MMVQ 1
+#define VDR_IQ1_M_Q8_1_MMQ 1
+
+static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq1_m * bq1 = (const block_iq1_m *) vbq + kbx;
+
+ const int qs_packed = get_int_b4(bq1->qs, iqs);
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
+
+ int sumi[2] = {0};
+ float sumf[2] = {0.0f};
+#pragma unroll
+ for (int l0 = 0; l0 < 8; l0 += 2) {
+ const int qhl = bq1->qh[2*iqs + l0/4] >> (4 * ((l0/2) % 2));
+
+ const int grid = iq1s_grid_gpu[qs[l0/2] | ((qhl & 0x07) << 8)];
+
+ const int grid0 = (grid >> 0) & 0x0F0F0F0F;
+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;
+
+ const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
+ const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
+
+ sumi[l0/4] = ggml_cuda_dp4a(grid0, u0, sumi[l0/4]);
+ sumi[l0/4] = ggml_cuda_dp4a(grid1, u1, sumi[l0/4]);
+
+ const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f*IQ1M_DELTA/0x08);
+ int sumy = 0;
+ sumy = ggml_cuda_dp4a(u0, 0x01010101, sumy);
+ sumy = ggml_cuda_dp4a(u1, 0x01010101, sumy);
+ sumf[l0/4] += delta*sumy;
+ }
+
+ const uint16_t * sc = (const uint16_t *) bq1->scales;
+
+ iq1m_scale_t scale;
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
+ const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);
+
+ const int tmp = sc[iqs/2] >> (6*(iqs%2));
+ const int sc0 = 2*((tmp >> 0) & 0x07) + 1;
+ const int sc1 = 2*((tmp >> 3) & 0x07) + 1;
+ return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);
+}
+
+#define VDR_IQ4_NL_Q8_1_MMVQ 2
+#define VDR_IQ4_NL_Q8_1_MMQ 4
+
+static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq4_nl * bq4 = (const block_iq4_nl *) vbq + kbx;
+
+ const int * q8 = (const int *) bq8_1->qs + iqs;
+
+ int sumi = 0;
+#pragma unroll
+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
+ const int aux_q4 = get_int_b2(bq4->qs, iqs + l);
+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);
+
+ sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);
+ sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);
+ }
+
+ const float d = __half2float(bq4->d) * __low2float(bq8_1->ds);
+ return d * sumi;
+}
+
+#define VDR_IQ4_XS_Q8_1_MMVQ 4
+#define VDR_IQ4_XS_Q8_1_MMQ 4
+
+static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+
+ const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq + kbx;
+
+ int sumi = 0;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int aux_q4 = get_int_b4(bq4->qs, iqs + j);
+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);
+
+ const int u0 = get_int_b4(bq8_1[iqs/4].qs, j + 0);
+ const int u1 = get_int_b4(bq8_1[iqs/4].qs, j + 4);
+
+ sumi = ggml_cuda_dp4a(v.x, u0, sumi);
+ sumi = ggml_cuda_dp4a(v.y, u1, sumi);
+ }
+
+ const int ls = ((bq4->scales_l[iqs/8] >> (iqs & 0x04)) & 0x0F) | (((bq4->scales_h >> (iqs/2)) & 0x03) << 4);
+ sumi *= ls - 32;
+
+ const float d = __half2float(bq4->d) * __low2float(bq8_1[iqs/4].ds);
+ return d * sumi;
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/cuda.h b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/cuda.h
new file mode 100644
index 00000000..323c9801
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/cuda.h
@@ -0,0 +1,28 @@
+#pragma once
+
+#include <cuda_runtime.h>
+#include <cuda.h>
+#include <cublas_v2.h>
+#include <cuda_bf16.h>
+#include <cuda_fp16.h>
+
+#ifdef GGML_USE_NCCL
+#include <nccl.h>
+#endif // GGML_USE_NCCL
+
+#if CUDART_VERSION >= 11080
+#include <cuda_fp8.h>
+#define FP8_AVAILABLE
+#endif // CUDART_VERSION >= 11080
+
+#if CUDART_VERSION >= 12080
+#include <cuda_fp4.h>
+#endif // CUDART_VERSION >= 12080
+
+#if CUDART_VERSION < 11020
+#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED
+#define CUBLAS_TF32_TENSOR_OP_MATH CUBLAS_TENSOR_OP_MATH
+#define CUBLAS_COMPUTE_16F CUDA_R_16F
+#define CUBLAS_COMPUTE_32F CUDA_R_32F
+#define cublasComputeType_t cudaDataType_t
+#endif // CUDART_VERSION < 11020
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/hip.h b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/hip.h
new file mode 100644
index 00000000..da7e3da0
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/hip.h
@@ -0,0 +1,313 @@
+#pragma once
+
+#define HIP_DISABLE_WARP_SYNC_BUILTINS 1
+#include <hip/hip_runtime.h>
+#include <hipblas/hipblas.h>
+#include <hip/hip_fp16.h>
+#include <hip/hip_bf16.h>
+
+#ifdef GGML_USE_NCCL
+#include <rccl/rccl.h>
+#endif // GGML_USE_NCCL
+
+
+#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT
+#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT
+#define CUBLAS_OP_N HIPBLAS_OP_N
+#define CUBLAS_OP_T HIPBLAS_OP_T
+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
+#define CUBLAS_TF32_TENSOR_OP_MATH 0
+#define CUDA_R_16F HIPBLAS_R_16F
+#define CUDA_R_16BF HIPBLAS_R_16B
+#define CUDA_R_32F HIPBLAS_R_32F
+#define CUBLAS_SIDE_RIGHT HIPBLAS_SIDE_RIGHT
+#define CUBLAS_FILL_MODE_UPPER HIPBLAS_FILL_MODE_UPPER
+#define CUBLAS_DIAG_NON_UNIT HIPBLAS_DIAG_NON_UNIT
+#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED hipDeviceAttributeVirtualMemoryManagementSupported
+#define CU_MEM_ALLOC_GRANULARITY_RECOMMENDED hipMemAllocationGranularityRecommended
+#define CU_MEM_ALLOCATION_TYPE_PINNED hipMemAllocationTypePinned
+#define CU_MEM_LOCATION_TYPE_DEVICE hipMemLocationTypeDevice
+#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE hipMemAccessFlagsProtReadWrite
+#define CU_CHECK(fn) {hipError_t err = fn; if(err != hipSuccess) { GGML_ABORT("HipVMM Failure: %s\n", hipGetErrorString(err)); }}
+#define __shfl_sync(mask, var, laneMask, width) __shfl(var, laneMask, width)
+#define __shfl_up_sync(mask, var, laneMask, width) __shfl_up(var, laneMask, width)
+#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)
+#define __all_sync(mask, var) __all(var)
+#define __any_sync(mask, var) __any(var)
+#define cublasStrsmBatched hipblasStrsmBatched
+#define cublasCreate hipblasCreate
+#define cublasDestroy hipblasDestroy
+#define cublasGemmEx hipblasGemmEx
+#define cublasGemmBatchedEx hipblasGemmBatchedEx
+#define cublasGemmStridedBatchedEx hipblasGemmStridedBatchedEx
+#define cublasHandle_t hipblasHandle_t
+#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS
+#define cublasSetStream hipblasSetStream
+#define cublasSgemm hipblasSgemm
+#define cublasSgemmBatched hipblasSgemmBatched
+#define cublasSgemmStridedBatched hipblasSgemmStridedBatched
+#define cublasStatus_t hipblasStatus_t
+#define cublasOperation_t hipblasOperation_t
+#define cudaDevAttrCooperativeLaunch hipDeviceAttributeCooperativeLaunch
+#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer
+#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess
+#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess
+#define cudaDeviceGetAttribute hipDeviceGetAttribute
+#define cudaDeviceGetPCIBusId hipDeviceGetPCIBusId
+#define cudaDeviceProp hipDeviceProp_t
+#define cudaDeviceSynchronize hipDeviceSynchronize
+#define cudaError_t hipError_t
+#define cudaErrorMemoryAllocation hipErrorOutOfMemory
+#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled
+#define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled
+#define cudaEventCreateWithFlags hipEventCreateWithFlags
+#define cudaEventDisableTiming hipEventDisableTiming
+#define cudaEventRecord hipEventRecord
+#define cudaEventSynchronize hipEventSynchronize
+#define cudaEvent_t hipEvent_t
+#define cudaEventDestroy hipEventDestroy
+#define cudaFree hipFree
+#define cudaFreeHost hipHostFree
+#define cudaGetDevice hipGetDevice
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetDeviceProperties hipGetDeviceProperties
+#define cudaGetErrorString hipGetErrorString
+#define cudaGetLastError hipGetLastError
+#define cudaHostAlloc hipHostMalloc
+#define cudaHostAllocPortable hipHostMallocPortable
+#define cudaHostAllocMapped hipHostMallocMapped
+#define cudaHostGetDevicePointer hipHostGetDevicePointer
+#define cudaHostRegister hipHostRegister
+#define cudaHostRegisterPortable hipHostRegisterPortable
+#define cudaHostRegisterReadOnly hipHostRegisterReadOnly
+#define cudaHostUnregister hipHostUnregister
+#define cudaLaunchCooperativeKernel hipLaunchCooperativeKernel
+#define cudaLaunchHostFunc hipLaunchHostFunc
+#define cudaMalloc hipMalloc
+#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size, hipHostMallocDefault)
+#define cudaMallocManaged hipMallocManaged
+#define cudaMemAdvise hipMemAdvise
+#define cudaMemcpy hipMemcpy
+#define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyPeerAsync hipMemcpyPeerAsync
+#define cudaMemcpy2DAsync hipMemcpy2DAsync
+#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
+#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost
+#define cudaMemcpyHostToDevice hipMemcpyHostToDevice
+#define cudaMemcpyKind hipMemcpyKind
+#define cudaMemset hipMemset
+#define cudaMemsetAsync hipMemsetAsync
+#define cudaMemGetInfo hipMemGetInfo
+#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize
+#define cudaSetDevice hipSetDevice
+#define cuDeviceGet hipDeviceGet
+#define CUdevice hipDevice_t
+#define CUdeviceptr hipDeviceptr_t
+#define cuMemUnmap hipMemUnmap
+#define CUmemAccessDesc hipMemAccessDesc
+#define cuMemAddressFree hipMemAddressFree
+#define cuMemRelease hipMemRelease
+#define CUmemGenericAllocationHandle hipMemGenericAllocationHandle_t
+#define cuMemCreate hipMemCreate
+#define cuMemAddressReserve hipMemAddressReserve
+#define cuMemMap hipMemMap
+#define cuMemSetAccess hipMemSetAccess
+#define cuMemGetAllocationGranularity hipMemGetAllocationGranularity
+#define CUmemAllocationProp hipMemAllocationProp
+#define cuDeviceGetAttribute hipDeviceGetAttribute
+#define cudaStreamCreateWithFlags hipStreamCreateWithFlags
+#define cudaStreamDestroy hipStreamDestroy
+#define cudaStreamFireAndForget hipStreamFireAndForget
+#define cudaStreamNonBlocking hipStreamNonBlocking
+#define cudaStreamPerThread hipStreamPerThread
+#define cudaStreamSynchronize hipStreamSynchronize
+#define cudaStreamWaitEvent hipStreamWaitEvent
+#define cudaGraphExec_t hipGraphExec_t
+#define cudaGraphNode_t hipGraphNode_t
+#define cudaKernelNodeParams hipKernelNodeParams
+#define cudaKernelNodeParams hipKernelNodeParams
+#define cudaGraphExecDestroy hipGraphExecDestroy
+#define cudaGraphLaunch hipGraphLaunch
+#define cudaErrorGraphExecUpdateFailure hipErrorGraphExecUpdateFailure
+#define cudaGraphExecUpdateResult hipGraphExecUpdateResult
+#define cudaGraphNodeType hipGraphNodeType
+#define cudaGraphNodeTypeKernel hipGraphNodeTypeKernel
+#define cudaGraphInstantiate hipGraphInstantiate
+#define cudaStreamEndCapture hipStreamEndCapture
+#define cudaGraphDestroy hipGraphDestroy
+#define cudaGraphKernelNodeSetParams hipGraphKernelNodeSetParams
+#define cudaErrorInvalidDeviceFunction hipErrorInvalidDeviceFunction
+#define cudaGraphKernelNodeGetParams hipGraphKernelNodeGetParams
+#define cudaGraphNodeGetType hipGraphNodeGetType
+#define cudaGraphGetNodes hipGraphGetNodes
+#define cudaGraphExecUpdate hipGraphExecUpdate
+#define cudaStreamCaptureModeRelaxed hipStreamCaptureModeRelaxed
+#define cudaStreamBeginCapture hipStreamBeginCapture
+#define cudaGraph_t hipGraph_t
+#define cudaStream_t hipStream_t
+#define cudaSuccess hipSuccess
+#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor
+#define cudaFuncSetAttribute hipFuncSetAttribute
+#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize
+#define __trap() do { abort(); __builtin_unreachable(); } while(0)
+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
+#define CUBLAS_STATUS_NOT_INITIALIZED HIPBLAS_STATUS_NOT_INITIALIZED
+#define CUBLAS_STATUS_ALLOC_FAILED HIPBLAS_STATUS_ALLOC_FAILED
+#define CUBLAS_STATUS_INVALID_VALUE HIPBLAS_STATUS_INVALID_VALUE
+#define CUBLAS_STATUS_ARCH_MISMATCH HIPBLAS_STATUS_ARCH_MISMATCH
+#define CUBLAS_STATUS_MAPPING_ERROR HIPBLAS_STATUS_MAPPING_ERROR
+#define CUBLAS_STATUS_EXECUTION_FAILED HIPBLAS_STATUS_EXECUTION_FAILED
+#define CUBLAS_STATUS_INTERNAL_ERROR HIPBLAS_STATUS_INTERNAL_ERROR
+#define CUBLAS_STATUS_NOT_SUPPORTED HIPBLAS_STATUS_NOT_SUPPORTED
+
+#if HIP_VERSION >= 60500000
+#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F
+#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F
+#define cublasComputeType_t hipblasComputeType_t
+#define cudaDataType_t hipDataType
+#else
+#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F
+#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F
+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_R_32F
+#define cublasComputeType_t hipblasDatatype_t
+#define cudaDataType_t hipblasDatatype_t
+#endif // HIP_VERSION >= 6050000
+
+#if !defined(__HIP_PLATFORM_AMD__)
+#error "The HIP backend supports only AMD targets"
+#endif // !defined(__HIP_PLATFORM_AMD__)
+
+#define __CUDA_ARCH__ 1300
+
+#if defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__)
+#define GCN5
+#endif // defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__)
+
+#if defined(__gfx803__)
+#define GCN4
+#endif // defined(__gfx803__)
+
+#if defined(GCN5) || defined(GCN4)
+#define GCN
+#endif // defined(GCN5) || defined(GCN4)
+
+#if defined(__gfx950__)
+#define CDNA4
+#endif // defined(__gfx950__)
+
+#if defined(__gfx942__)
+#define CDNA3
+#endif // defined(__gfx942__)
+
+#if defined(__gfx90a__)
+#define CDNA2
+#endif // defined(__gfx90a__)
+
+#if defined(__gfx908__)
+#define CDNA1
+#endif // defined(__gfx908__)
+
+#if defined(CDNA4) || defined(CDNA3) || defined(CDNA2) || defined(CDNA1)
+#define CDNA // For the entire family
+#endif // defined(CDNA4) || defined(CDNA3) || defined(CDNA2) || defined(CDNA1)
+
+#if defined(__GFX12__)
+#define RDNA4
+#endif // defined(__GFX12__)
+
+#if defined(__GFX11__)
+#define RDNA3
+#endif // defined(__GFX11__)
+
+#if defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)
+#define RDNA3_5
+#endif // defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)
+
+#if defined(RDNA3) && !defined(RDNA3_5)
+#define RDNA3_0
+#endif // defined(RDNA3) && !defined(RDNA3_5)
+
+#if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \
+ defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || defined(__gfx1037__)
+#define RDNA2
+#endif
+
+#if defined(__gfx1010__) || defined(__gfx1012__)
+#define RDNA1
+#endif // defined(__gfx1010__) || defined(__gfx1012__)
+
+#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
+#define RDNA // For the entire family
+#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
+
+#ifndef __has_builtin
+ #define __has_builtin(x) 0
+#endif
+
+typedef __hip_bfloat16 nv_bfloat16;
+typedef __hip_bfloat162 nv_bfloat162;
+
+#if HIP_VERSION >= 60300000
+#include <hip/hip_fp8.h>
+typedef __hip_fp8_e4m3 __nv_fp8_e4m3;
+#define FP8_AVAILABLE
+#endif // HIP_VERSION >= 60300000
+
+typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));
+typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));
+static __device__ __forceinline__ int __vsubss4(const int a, const int b) {
+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
+#if __has_builtin(__builtin_elementwise_sub_sat)
+ const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);
+ return reinterpret_cast<const int &>(c);
+#else
+ int8x4_t c;
+ int16_t tmp;
+#pragma unroll
+ for (int i = 0; i < 4; i++) {
+ tmp = va[i] - vb[i];
+ if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();
+ if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();
+ c[i] = tmp;
+ }
+ return reinterpret_cast<int &>(c);
+#endif // __has_builtin(__builtin_elementwise_sub_sat)
+}
+
+static __device__ __forceinline__ int __vsub4(const int a, const int b) {
+ // do some small modifications to a and b to make the subtraction not underflow
+ const unsigned int a_large = a | 0x80808080;
+ const unsigned int b_small = b & 0x7f7f7f7f;
+ const unsigned int result_low_7bits = a_large - b_small;
+
+ // if two ops share the same high bit, we should flip the high bit of the result
+ const unsigned int should_flip_high_1bit = (a ^ ~b) & 0x80808080;
+
+ return result_low_7bits ^ should_flip_high_1bit;
+}
+
+static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) {
+ const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);
+ const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);
+ unsigned int c;
+ uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ vc[i] = va[i] == vb[i] ? 0xff : 0x00;
+ }
+ return c;
+}
+
+static __device__ __forceinline__ unsigned int __vcmpne4(unsigned int a, unsigned int b) {
+ const unsigned int x = a ^ b;
+
+ // any non-equal bit in a byte will set the high bit of that byte here
+ // the addition will not overflow in the byte as op1 and op2 are both less than 0x80
+ const unsigned int ne_low_7bits = ((x & 0x7f7f7f7f) + 0x7f7f7f7f) & 0x80808080;
+ const unsigned int ne_high_1bit = x & 0x80808080;
+ const unsigned int ne_any_bit = ne_low_7bits | ne_high_1bit;
+
+ return (ne_any_bit >> 7) * 0xff;
+}
diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/musa.h b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/musa.h
new file mode 100644
index 00000000..22126dec
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/musa.h
@@ -0,0 +1,161 @@
+#pragma once
+
+#include <musa_runtime.h>
+#include <musa.h>
+#include <mublas.h>
+#include <musa_bf16.h>
+#include <musa_fp16.h>
+
+#ifdef __MUSA_ARCH__
+#define __CUDA_ARCH__ 1300 // GGML_CUDA_CC_RUBIN
+#endif // __MUSA_ARCH__
+
+#define CUBLAS_COMPUTE_16F CUDA_R_16F
+#define CUBLAS_COMPUTE_32F CUDA_R_32F
+#define CUBLAS_COMPUTE_32F_FAST_16F MUBLAS_COMPUTE_32F_FAST_16F
+#define CUBLAS_GEMM_DEFAULT MUBLAS_GEMM_DEFAULT
+#define CUBLAS_GEMM_DEFAULT_TENSOR_OP MUBLAS_GEMM_DEFAULT
+#define CUBLAS_OP_N MUBLAS_OP_N
+#define CUBLAS_OP_T MUBLAS_OP_T
+#define CUBLAS_DEFAULT_MATH MUBLAS_DEFAULT_MATH
+#define CUBLAS_SIDE_RIGHT MUBLAS_SIDE_RIGHT
+#define CUBLAS_FILL_MODE_UPPER MUBLAS_FILL_MODE_UPPER
+#define CUBLAS_DIAG_NON_UNIT MUBLAS_DIAG_NON_UNIT
+#define CUBLAS_STATUS_SUCCESS MUBLAS_STATUS_SUCCESS
+#define CUBLAS_TF32_TENSOR_OP_MATH MUBLAS_TENSOR_OP_MATH
+#define CUDA_R_16F MUSA_R_16F
+#define CUDA_R_16BF MUSA_R_16BF
+#define CUDA_R_32F MUSA_R_32F
+#define cublasStrsmBatched mublasStrsmBatched
+#define cublasComputeType_t cudaDataType_t
+#define cublasCreate mublasCreate
+#define cublasDestroy mublasDestroy
+#define cublasGemmEx mublasGemmEx
+#define cublasGemmBatchedEx mublasGemmBatchedEx
+#define cublasGemmStridedBatchedEx mublasGemmStridedBatchedEx
+#define cublasHandle_t mublasHandle_t
+#define cublasSetMathMode mublasSetMathMode
+#define cublasSetStream mublasSetStream
+#define cublasSgemm mublasSgemm
+#define cublasSgemmBatched mublasSgemmBatched
+#define cublasSgemmStridedBatched mublasSgemmStridedBatched
+#define cublasStatus_t mublasStatus_t
+#define cublasOperation_t mublasOperation_t
+#define cublasGetStatusString mublasGetStatusString
+#define cudaDataType_t musaDataType_t
+#define cudaDeviceCanAccessPeer musaDeviceCanAccessPeer
+#define cudaDeviceDisablePeerAccess musaDeviceDisablePeerAccess
+#define cudaDeviceEnablePeerAccess musaDeviceEnablePeerAccess
+#define cudaDeviceGetPCIBusId musaDeviceGetPCIBusId
+#define cudaDeviceProp musaDeviceProp
+#define cudaDeviceSynchronize musaDeviceSynchronize
+#define cudaDeviceGetAttribute musaDeviceGetAttribute
+#define cudaError_t musaError_t
+#define cudaErrorMemoryAllocation musaErrorMemoryAllocation
+#define cudaErrorPeerAccessAlreadyEnabled musaErrorPeerAccessAlreadyEnabled
+#define cudaErrorPeerAccessNotEnabled musaErrorPeerAccessNotEnabled
+#define cudaEventCreateWithFlags musaEventCreateWithFlags
+#define cudaEventDisableTiming musaEventDisableTiming
+#define cudaEventRecord musaEventRecord
+#define cudaEventSynchronize musaEventSynchronize
+#define cudaEvent_t musaEvent_t
+#define cudaEventDestroy musaEventDestroy
+#define cudaFree musaFree
+#define cudaFreeHost musaFreeHost
+#define cudaGetDevice musaGetDevice
+#define cudaGetDeviceCount musaGetDeviceCount
+#define cudaGetDeviceProperties musaGetDeviceProperties
+#define cudaGetErrorString musaGetErrorString
+#define cudaGetLastError musaGetLastError
+#define cudaHostRegister musaHostRegister
+#define cudaHostRegisterPortable musaHostRegisterPortable
+#define cudaHostRegisterReadOnly musaHostRegisterReadOnly
+#define cudaHostUnregister musaHostUnregister
+#define cudaLaunchCooperativeKernel musaLaunchCooperativeKernel
+#define cudaLaunchHostFunc musaLaunchHostFunc
+#define cudaMalloc musaMalloc
+#define cudaMallocHost musaMallocHost
+#define cudaMallocManaged musaMallocManaged
+#define cudaMemcpy musaMemcpy
+#define cudaMemcpyAsync musaMemcpyAsync
+#define cudaMemcpyPeerAsync musaMemcpyPeerAsync
+#define cudaMemcpy2DAsync musaMemcpy2DAsync
+#define cudaMemcpyDeviceToDevice musaMemcpyDeviceToDevice
+#define cudaMemcpyDeviceToHost musaMemcpyDeviceToHost
+#define cudaMemcpyHostToDevice musaMemcpyHostToDevice
+#define cudaMemcpyKind musaMemcpyKind
+#define cudaMemset musaMemset
+#define cudaMemsetAsync musaMemsetAsync
+#define cudaMemGetInfo musaMemGetInfo
+#define cudaOccupancyMaxPotentialBlockSize musaOccupancyMaxPotentialBlockSize
+#define cudaSetDevice musaSetDevice
+#define cudaStreamCreateWithFlags musaStreamCreateWithFlags
+#define cudaStreamDestroy musaStreamDestroy
+#define cudaStreamFireAndForget musaStreamFireAndForget
+#define cudaStreamNonBlocking musaStreamNonBlocking
+#define cudaStreamPerThread musaStreamPerThread
+#define cudaStreamSynchronize musaStreamSynchronize
+#define cudaStreamWaitEvent musaStreamWaitEvent
+#define cudaStream_t musaStream_t
+#define cudaSuccess musaSuccess
+
+// Additional mappings for MUSA virtual memory pool
+#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED MU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED
+#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE MU_MEM_ACCESS_FLAGS_PROT_READWRITE
+#define CU_MEM_ALLOC_GRANULARITY_RECOMMENDED MU_MEM_ALLOC_GRANULARITY_RECOMMENDED
+#define CU_MEM_ALLOCATION_TYPE_PINNED MU_MEM_ALLOCATION_TYPE_PINNED
+#define CU_MEM_LOCATION_TYPE_DEVICE MU_MEM_LOCATION_TYPE_DEVICE
+#define CUdevice MUdevice
+#define CUdeviceptr MUdeviceptr
+#define CUmemAccessDesc MUmemAccessDesc
+#define CUmemAllocationProp MUmemAllocationProp
+#define CUmemGenericAllocationHandle MUmemGenericAllocationHandle
+#define cuDeviceGet muDeviceGet
+#define cuDeviceGetAttribute muDeviceGetAttribute
+#define cuMemAddressFree muMemAddressFree
+#define cuMemAddressReserve muMemAddressReserve
+#define cuMemCreate muMemCreate
+#define cuMemGetAllocationGranularity muMemGetAllocationGranularity
+#define cuMemMap muMemMap
+#define cuMemRelease muMemRelease
+#define cuMemSetAccess muMemSetAccess
+#define cuMemUnmap muMemUnmap
+#define cudaDevAttrCooperativeLaunch musaDevAttrCooperativeLaunch
+#define cudaFuncAttributeMaxDynamicSharedMemorySize musaFuncAttributeMaxDynamicSharedMemorySize
+#define cudaFuncSetAttribute musaFuncSetAttribute
+#define cudaMemcpy3DPeerParms musaMemcpy3DPeerParms
+#define make_cudaExtent make_musaExtent
+#define make_cudaPitchedPtr make_musaPitchedPtr
+
+// Additional mappings for MUSA graphs
+#define CUDA_SUCCESS MUSA_SUCCESS
+#define CUresult MUresult
+#define cuGetErrorString muGetErrorString
+#define cudaErrorGraphExecUpdateFailure musaErrorGraphExecUpdateFailure
+#define cudaErrorInvalidDeviceFunction musaErrorInvalidDeviceFunction
+#define cudaGraphDestroy musaGraphDestroy
+#define cudaGraphExecDestroy musaGraphExecDestroy
+#define cudaGraphExec_t musaGraphExec_t
+#define cudaGraphExecUpdate musaGraphExecUpdate
+#define cudaGraphExecUpdateResult musaGraphExecUpdateResult
+#define cudaGraphGetNodes musaGraphGetNodes
+#define cudaGraphInstantiate musaGraphInstantiate
+#define cudaGraphKernelNodeGetParams musaGraphKernelNodeGetParams
+#define cudaGraphKernelNodeSetParams musaGraphKernelNodeSetParams
+#define cudaGraphLaunch musaGraphLaunch
+#define cudaGraphNodeGetType musaGraphNodeGetType
+#define cudaGraphNode_t musaGraphNode_t
+#define cudaGraphNodeType musaGraphNodeType
+#define cudaGraphNodeTypeKernel musaGraphNodeTypeKernel
+#define cudaGraph_t musaGraph_t
+#define cudaKernelNodeParams musaKernelNodeParams
+#define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed
+#define cudaStreamBeginCapture musaStreamBeginCapture
+#define cudaStreamEndCapture musaStreamEndCapture
+#define cudaStreamCaptureStatus musaStreamCaptureStatus
+#define cudaStreamCaptureStatusNone musaStreamCaptureStatusNone
+#define cudaStreamIsCapturing musaStreamIsCapturing
+#define cudaOccupancyMaxActiveBlocksPerMultiprocessor musaOccupancyMaxActiveBlocksPerMultiprocessor
+
+typedef __mt_bfloat16 nv_bfloat16;
+typedef __mt_bfloat162 nv_bfloat162;
diff --git a/python/freetoken/kernel/csrc/ggml/include/ggml-alloc.h b/python/freetoken/kernel/csrc/ggml/include/ggml-alloc.h
new file mode 100644
index 00000000..81cce2e2
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/include/ggml-alloc.h
@@ -0,0 +1,87 @@
+#pragma once
+
+#include "ggml.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
+typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
+typedef struct ggml_backend * ggml_backend_t;
+
+// Tensor allocator
+struct ggml_tallocr {
+ ggml_backend_buffer_t buffer;
+ void * base;
+ size_t alignment;
+ size_t offset;
+};
+
+GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer);
+GGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor);
+
+// Graph allocator
+/*
+ Example usage:
+ ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type());
+
+ // optional: create a worst-case graph and reserve the buffers to avoid reallocations
+ ggml_gallocr_reserve(galloc, build_graph(max_batch));
+
+ // allocate the graph
+ struct ggml_cgraph * graph = build_graph(batch);
+ ggml_gallocr_alloc_graph(galloc, graph);
+
+ printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0));
+
+ // evaluate the graph
+ ggml_backend_graph_compute(backend, graph);
+*/
+
+// special tensor flags for use with the graph allocator:
+// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses
+// ggml_set_output(): output tensors are never freed and never overwritten
+
+typedef struct ggml_gallocr * ggml_gallocr_t;
+
+GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft);
+GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs);
+GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);
+
+// pre-allocate buffers from a measure graph - does not allocate or modify the graph
+// call with a worst-case graph to avoid buffer reallocations
+// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed
+// returns false if the buffer allocation failed
+// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes
+GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);
+GGML_API void ggml_gallocr_reserve_n_size(
+ ggml_gallocr_t galloc,
+ struct ggml_cgraph * graph,
+ const int * node_buffer_ids,
+ const int * leaf_buffer_ids,
+ size_t * sizes);
+GGML_API bool ggml_gallocr_reserve_n(
+ ggml_gallocr_t galloc,
+ struct ggml_cgraph * graph,
+ const int * node_buffer_ids,
+ const int * leaf_buffer_ids);
+
+// automatic reallocation if the topology changes when using a single buffer
+// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)
+GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph);
+
+GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id);
+
+// Utils
+// Create a buffer and allocate all the tensors in a ggml_context
+
+// returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft. returns 0 on failure
+GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);
+// returns NULL on failure or if all tensors in ctx are already allocated or zero-sized
+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);
+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/python/freetoken/kernel/csrc/ggml/include/ggml-backend.h b/python/freetoken/kernel/csrc/ggml/include/ggml-backend.h
new file mode 100644
index 00000000..7a65d812
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/include/ggml-backend.h
@@ -0,0 +1,463 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-alloc.h"
+
+#ifdef GGML_BACKEND_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef GGML_BACKEND_BUILD
+# define GGML_BACKEND_API __declspec(dllexport) extern
+# else
+# define GGML_BACKEND_API __declspec(dllimport) extern
+# endif
+# else
+# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern
+# endif
+#else
+# define GGML_BACKEND_API extern
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
+ typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
+ typedef struct ggml_backend_event * ggml_backend_event_t;
+ typedef struct ggml_backend * ggml_backend_t;
+ typedef void * ggml_backend_graph_plan_t;
+ typedef struct ggml_backend_reg * ggml_backend_reg_t;
+ typedef struct ggml_backend_device * ggml_backend_dev_t;
+
+
+ //
+ // Backend buffer type
+ //
+
+ GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);
+ GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);
+ GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);
+ GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);
+ GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);
+ GGML_API size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);
+ GGML_API size_t ggml_backend_buft_get_alloc_size_n(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);
+ GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);
+ GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft);
+
+ //
+ // Backend buffer
+ //
+
+ enum ggml_backend_buffer_usage {
+ GGML_BACKEND_BUFFER_USAGE_ANY = 0,
+ GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
+ GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,
+ };
+
+ GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);
+ GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);
+ GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);
+ GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);
+ GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+ GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);
+ GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);
+ GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor);
+ GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);
+ GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);
+ GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
+ GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer);
+ GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);
+ GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);
+
+ // tensor copy between different backends
+ GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst);
+
+ //
+ // Backend (stream)
+ //
+
+ GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);
+ GGML_API const char * ggml_backend_name(ggml_backend_t backend);
+ GGML_API void ggml_backend_free(ggml_backend_t backend);
+
+ GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);
+ GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);
+ GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);
+ GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);
+
+ GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+ GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+ GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+ GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+
+ // "offset" refers to the offset in tensor->data for setting/getting data
+ GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+ GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+ GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+ GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+ GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);
+
+ GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
+
+ GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);
+ GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+
+ GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+ GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
+ GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
+
+ // NOTE: will be removed, use device version instead
+ GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
+ GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
+ GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);
+
+ // asynchronous copy
+ // the copy is performed after all the currently queued operations in backend_src
+ // backend_dst will wait for the copy to complete before performing other operations
+ // automatic fallback to sync copy if async is not supported
+ GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
+
+ GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend);
+
+ //
+ // Events
+ //
+
+ GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device);
+ GGML_API void ggml_backend_event_free(ggml_backend_event_t event);
+ GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend);
+ GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);
+ GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event);
+
+ //
+ // Backend device
+ //
+
+ enum ggml_backend_dev_type {
+ // CPU device using system memory
+ GGML_BACKEND_DEVICE_TYPE_CPU,
+ // GPU device using dedicated memory
+ GGML_BACKEND_DEVICE_TYPE_GPU,
+ // integrated GPU device using host memory
+ GGML_BACKEND_DEVICE_TYPE_IGPU,
+ // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)
+ GGML_BACKEND_DEVICE_TYPE_ACCEL,
+ // "meta" device wrapping multiple other devices for tensor parallelism
+ GGML_BACKEND_DEVICE_TYPE_META,
+ };
+
+ // functionality supported by the device
+ struct ggml_backend_dev_caps {
+ // asynchronous operations
+ bool async;
+ // pinned host buffer
+ bool host_buffer;
+ // creating buffers from host ptr
+ bool buffer_from_host_ptr;
+ // event synchronization
+ bool events;
+ // mmap is supported for loading
+ bool mmap_support;
+ };
+
+ // all the device properties
+ struct ggml_backend_dev_props {
+ // device name
+ const char * name;
+ // device description
+ const char * description;
+ // device free memory in bytes
+ size_t memory_free;
+ // device total memory in bytes
+ size_t memory_total;
+ // device type
+ enum ggml_backend_dev_type type;
+ // device id
+ // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0")
+ // if the id is unknown, this should be NULL
+ const char * device_id;
+ // device capabilities
+ struct ggml_backend_dev_caps caps;
+ };
+
+ GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device);
+ GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device);
+ GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total);
+ GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device);
+ GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props);
+ GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device);
+ GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params);
+ GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device);
+ GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device);
+ GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);
+
+ GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
+ GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);
+ GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
+
+ //
+ // Backend (reg)
+ //
+
+ GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg);
+ GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg);
+ GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index);
+ GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name);
+
+ // Common functions that may be obtained using ggml_backend_reg_get_proc_address
+
+ // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend)
+ typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends);
+ typedef void (*ggml_backend_comm_free_t)(void * comm_ctx);
+ typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);
+
+ // Split buffer type for tensor parallelism (old)
+ typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
+ // Set the number of threads for the backend
+ typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
+ // Get additional buffer types provided by the device (returns a NULL-terminated array)
+ typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
+ // Set the abort callback for the backend
+ typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
+ // Get a list of feature flags supported by the backend (returns a NULL-terminated array)
+ struct ggml_backend_feature {
+ const char * name;
+ const char * value;
+ };
+ typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);
+
+ //
+ // Backend registry
+ //
+
+ GGML_API void ggml_backend_register(ggml_backend_reg_t reg);
+
+ GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
+
+ // Backend (reg) enumeration
+ GGML_API size_t ggml_backend_reg_count(void);
+ GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index);
+ GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name);
+
+ // Device enumeration
+ GGML_API size_t ggml_backend_dev_count(void);
+ GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index);
+ GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name);
+ GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type);
+
+ // Direct backend (stream) initialization
+ // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params)
+ GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params);
+ // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params)
+ GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params);
+ // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)
+ GGML_API ggml_backend_t ggml_backend_init_best(void);
+
+ // Load a backend from a dynamic library and register it
+ GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);
+ // Unload a backend if loaded dynamically and unregister it
+ GGML_API void ggml_backend_unload(ggml_backend_reg_t reg);
+ // Load all known backends from dynamic libraries
+ GGML_API void ggml_backend_load_all(void);
+ GGML_API void ggml_backend_load_all_from_path(const char * dir_path);
+
+ //
+ // Backend scheduler
+ //
+
+ // The backend scheduler allows for multiple backend devices to be used together
+ // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
+ // The backends are selected based on:
+ // - the backend that supports the operation
+ // - the location of the pre-allocated tensors (e.g. the weights)
+ /*
+ Example usage:
+
+ // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
+ // preferably to run on the same backend as the buffer
+ ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+
+ sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true);
+
+ // initialize buffers from a max size graph (optional)
+ reserve_graph = build_graph(sched, max_batch_size);
+
+ // manually assign nodes to a backend (optional, should not be needed in most cases)
+ struct ggml_tensor * node = ggml_mul_mat(ctx, ...);
+ ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
+
+ ggml_backend_sched_reserve(sched, reserve_graph);
+
+ // compute
+ graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation
+ for (int i = 0; i < 10; ++i) {
+ ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically
+ }
+
+ // if there are graph inputs:
+ graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called)
+ ggml_backend_sched_reset(sched); // clear the allocation of the previous graph
+ ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it
+ ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors
+ ggml_backend_sched_graph_compute(sched, graph); // execute the graph
+
+ // as an alternative to the above it is also possible to assign the inputs to a dedicated context and
+ // allocate them statically via ggml_backend_alloc_ctx_tensors
+ }
+ */
+
+ typedef struct ggml_backend_sched * ggml_backend_sched_t;
+
+ // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback)
+ // when ask == true, the scheduler wants to know if the user wants to observe this node
+ // this allows the scheduler to batch nodes together in order to evaluate them in a single call
+ //
+ // when ask == false, the scheduler is passing the node tensor to the user for observation
+ // if the user returns false, the scheduler will cancel the graph compute
+ //
+ typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
+
+ // Callback while copying input weights of a split
+ // if the user returns false the scheduler simply copies the entire weight
+ // the callback is called only for input weights in host buffers
+ // the callback is called after all non-weight inputs of the split have been copied
+ // `src` is the tensor in the previous split
+ // `dst` is the copy of `src` in the split
+ // `graph` is the compute graph of the split
+ typedef bool (*ggml_backend_sched_copy_callback)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst, struct ggml_cgraph * graph, void * user_data);
+
+ // Callback when the buffers reserved for a graph are not sufficient and have to be reserved again
+ // it happens on the graph allocation path and costs a synchronization of the backends
+ // `sched` is the scheduler
+ // `graph` is the compute graph being allocated
+ // `backend_ids_changed` is true if the backend assignment of the graph nodes changed since the last allocation
+ // `graph_size_changed` is true if the graph size changed since the last allocation
+ typedef void (*ggml_backend_sched_realloc_callback)(ggml_backend_sched_t sched, struct ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data);
+
+ // Initialize a backend scheduler, backends with low index are given priority over backends with high index
+ GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);
+ GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);
+
+ // Initialize backend buffers from a measure graph
+ GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes);
+ GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success
+
+ GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched);
+ GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i);
+
+ // Get the number of splits of the last graph
+ GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);
+ GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);
+
+ GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend);
+ GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);
+
+ GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);
+ GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);
+
+ // Split graph without allocating it
+ GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+
+ // Allocate and compute graph on the backend scheduler
+ GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success
+ GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+ GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+ GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
+
+ // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.
+ // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.
+ // The correct way to use this API is to discard the deallocated tensors and create new ones.
+ GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);
+
+ // Set a callback to be called for each resulting node during graph compute
+ GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
+
+ // Set a callback to be called when the inputs weights of a split are being copied
+ GGML_API void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data);
+
+ // Set a callback to be called when the compute buffers of a graph are being reallocated
+ GGML_API void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data);
+
+ //
+ // Meta backend
+ //
+
+#define GGML_BACKEND_META_MAX_DEVICES 16
+
+ enum ggml_backend_meta_split_axis {
+ // tensor split by tensor dimensions:
+ GGML_BACKEND_SPLIT_AXIS_0 = 0,
+ GGML_BACKEND_SPLIT_AXIS_1 = 1,
+ GGML_BACKEND_SPLIT_AXIS_2 = 2,
+ GGML_BACKEND_SPLIT_AXIS_3 = 3,
+
+ GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends
+ GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum
+
+ // for internal bookkeeping only:
+ GGML_BACKEND_SPLIT_AXIS_NONE = 98,
+ GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99,
+ };
+ GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis);
+
+ struct ggml_backend_meta_split_state {
+ enum ggml_backend_meta_split_axis axis;
+
+ // for tensors with axis >= 0 && axis < GGML_MAX_DIMS:
+ // - each device has a slice of the tensor along the split axis
+ // - most tensors have n_segments == 1 and a contiguous slice of the tensor data
+ // - some tensors have an inhomogenenous data layout along the split axis,
+ // those tensors are divided into segments which are each individually split across devices
+ // (this usually happens when multiple tensors are fused into a single one)
+ // - ne has one entry per segment and device and that segment repeats nr times,
+ // in total when accounting for repetitions the segments add up to ggml_tensor::ne for that axis,
+ // the outer/inner loops are over segments/devices like [seg0_dev0_r0, seg0_dev1_r0, seg0_dev0_r1, seg0_dev1_r1, seg1_dev0_r0, seg1_dev1_r0],
+ // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments
+ // that each need to be split individually across devices so that each device gets a slice of Q, K, and V,
+ // the Q matrix can be larger than the K and V matrices so this can either be expressed as 3 segments or as 2 segments
+ // where the segment for K/V repeats twice
+ int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES];
+ uint32_t nr[16];
+ uint32_t n_segments;
+ };
+
+ // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible:
+ typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata);
+
+ // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this:
+ // TODO: this looks a bit strange - a backend API creates a device. I think we should try
+ // express this as a backend registry functionality instead
+ GGML_API ggml_backend_dev_t ggml_backend_meta_device(
+ ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud);
+
+ //
+ // Utils
+ //
+
+ struct ggml_backend_graph_copy {
+ ggml_backend_buffer_t buffer;
+ struct ggml_context * ctx_allocated;
+ struct ggml_context * ctx_unallocated;
+ struct ggml_cgraph * graph;
+ };
+
+ // Copy a graph to a different backend
+ GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);
+ GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);
+
+ typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);
+
+ // Compare the output of two backends
+ GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes);
+
+ // Tensor initialization
+ GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
+ GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor);
+
+ // CPU buffer types are always available
+ GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
+ GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/python/freetoken/kernel/csrc/ggml/include/ggml-cuda.h b/python/freetoken/kernel/csrc/ggml/include/ggml-cuda.h
new file mode 100644
index 00000000..1cd81eea
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/include/ggml-cuda.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef GGML_USE_HIP
+#define GGML_CUDA_NAME "ROCm"
+#define GGML_CUBLAS_NAME "hipBLAS"
+#elif defined(GGML_USE_MUSA)
+#define GGML_CUDA_NAME "MUSA"
+#define GGML_CUBLAS_NAME "muBLAS"
+#else
+#define GGML_CUDA_NAME "CUDA"
+#define GGML_CUBLAS_NAME "cuBLAS"
+#endif
+#define GGML_CUDA_MAX_DEVICES 16
+
+// backend API
+GGML_BACKEND_API ggml_backend_t ggml_backend_cuda_init(int device);
+
+GGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend);
+
+// device buffer
+GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);
+
+// conduct allreduce operation between devices
+GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends);
+
+// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
+GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
+
+GGML_BACKEND_API int ggml_backend_cuda_get_device_count(void);
+GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
+GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
+
+GGML_BACKEND_API bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);
+GGML_BACKEND_API void ggml_backend_cuda_unregister_host_buffer(void * buffer);
+
+GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/python/freetoken/kernel/csrc/ggml/include/ggml.h b/python/freetoken/kernel/csrc/ggml/include/ggml.h
new file mode 100644
index 00000000..23ea6437
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/include/ggml.h
@@ -0,0 +1,3027 @@
+#pragma once
+
+//
+// GGML Tensor Library
+//
+// This documentation is still a work in progress.
+// If you wish some specific topics to be covered, feel free to drop a comment:
+//
+// https://github.com/ggml-org/whisper.cpp/issues/40
+//
+// ## Overview
+//
+// This library implements:
+//
+// - a set of tensor operations
+// - automatic differentiation
+// - basic optimization algorithms
+//
+// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
+// but is not limited to, the following:
+//
+// - linear regression
+// - support vector machines
+// - neural networks
+//
+// The library allows the user to define a certain function using the available tensor operations. This function
+// definition is represented internally via a computation graph. Each tensor operation in the function definition
+// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
+// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
+// using one of the available optimization algorithms.
+//
+// For example, here we define the function: f(x) = a*x^2 + b
+//
+// {
+// struct ggml_init_params params = {
+// .mem_size = 16*1024*1024,
+// .mem_buffer = NULL,
+// };
+//
+// // memory allocation happens here
+// struct ggml_context * ctx = ggml_init(params);
+//
+// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+//
+// ggml_set_param(ctx, x); // x is an input variable
+//
+// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+// struct ggml_tensor * x2 = ggml_mul(ctx, x, x);
+// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b);
+//
+// ...
+// }
+//
+// Notice that the function definition above does not involve any actual computation. The computation is performed only
+// when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
+//
+// {
+// ...
+//
+// struct ggml_cgraph * gf = ggml_new_graph(ctx);
+// ggml_build_forward_expand(gf, f);
+//
+// // set the input variable and parameter values
+// ggml_set_f32(x, 2.0f);
+// ggml_set_f32(a, 3.0f);
+// ggml_set_f32(b, 4.0f);
+//
+// ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
+//
+// printf("f = %f\n", ggml_get_f32_1d(f, 0));
+//
+// ...
+// }
+//
+// The actual computation is performed in the ggml_graph_compute() function.
+//
+// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
+// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
+// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
+// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was
+// actually needed.
+//
+// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
+// differentiation and optimization algorithms.
+//
+// The described approach allows to define the function graph once and then compute its forward or backward graphs
+// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way
+// the user can avoid the memory allocation overhead at runtime.
+//
+// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
+// citizens, but in theory the library can be extended to support FP8 and integer data types.
+//
+// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
+// and binary operations. Most of the available operations fall into one of these two categories. With time, it became
+// clear that the library needs to support more complex operations. The way to support these operations is not clear
+// yet, but a few examples are demonstrated in the following operations:
+//
+// - ggml_permute()
+// - ggml_conv_1d_1s()
+// - ggml_conv_1d_2s()
+//
+// For each tensor operator, the library implements a forward and backward computation function. The forward function
+// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
+// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
+// calculus class, or watch the following video:
+//
+// What is Automatic Differentiation?
+// https://www.youtube.com/watch?v=wG_nF1awSSY
+//
+//
+// ## Tensor data (struct ggml_tensor)
+//
+// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of
+// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
+// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
+//
+// {
+// struct ggml_tensor * c = ggml_add(ctx, a, b);
+//
+// assert(c->src[0] == a);
+// assert(c->src[1] == b);
+// }
+//
+// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the
+// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
+// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
+// permutation. All tensor operations have to take the stride into account and not assume that the tensor is
+// contiguous in memory.
+//
+// The data of the tensor is accessed via the "data" pointer. For example:
+//
+// {
+// const int nx = 2;
+// const int ny = 3;
+//
+// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);
+//
+// for (int y = 0; y < ny; y++) {
+// for (int x = 0; x < nx; x++) {
+// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
+// }
+// }
+//
+// ...
+// }
+//
+// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.
+//
+// ## The matrix multiplication operator (ggml_mul_mat)
+//
+// TODO
+//
+//
+// ## Multi-threading
+//
+// TODO
+//
+//
+// ## Overview of ggml.c
+//
+// TODO
+//
+//
+// ## SIMD optimizations
+//
+// TODO
+//
+//
+// ## Debugging ggml
+//
+// TODO
+//
+//
+
+#ifdef GGML_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef GGML_BUILD
+# define GGML_API __declspec(dllexport) extern
+# else
+# define GGML_API __declspec(dllimport) extern
+# endif
+# else
+# define GGML_API __attribute__ ((visibility ("default"))) extern
+# endif
+#else
+# define GGML_API extern
+#endif
+
+// TODO: support for clang
+#ifdef __GNUC__
+# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
+#elif defined(_MSC_VER)
+# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
+#else
+# define GGML_DEPRECATED(func, hint) func
+#endif
+
+#ifndef __GNUC__
+# define GGML_ATTRIBUTE_FORMAT(...)
+#elif defined(__MINGW32__) && !defined(__clang__)
+# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
+#else
+# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
+#endif
+
+#if defined(_WIN32) && !defined(_WIN32_WINNT)
+# define _WIN32_WINNT 0x0A00
+#endif
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#define GGML_FILE_MAGIC 0x67676d6c // "ggml"
+#define GGML_FILE_VERSION 2
+
+#define GGML_QNT_VERSION 2 // bump this on quantization format changes
+#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
+
+#define GGML_MAX_DIMS 4
+#define GGML_MAX_PARAMS 2048
+#define GGML_MAX_SRC 10
+#define GGML_MAX_N_THREADS 512
+#define GGML_MAX_OP_PARAMS 64
+
+#ifndef GGML_MAX_NAME
+# define GGML_MAX_NAME 64
+#endif
+
+#define GGML_DEFAULT_N_THREADS 4
+#define GGML_DEFAULT_GRAPH_SIZE 2048
+
+#if UINTPTR_MAX == 0xFFFFFFFF
+ #define GGML_MEM_ALIGN 4
+#elif defined(__EMSCRIPTEN__)
+// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm.
+// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.)
+// ref: https://github.com/ggml-org/llama.cpp/pull/18628
+ #define GGML_MEM_ALIGN 8
+#else
+ #define GGML_MEM_ALIGN 16
+#endif
+
+#define GGML_EXIT_SUCCESS 0
+#define GGML_EXIT_ABORTED 1
+
+// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726
+#define GGML_ROPE_TYPE_NORMAL 0
+#define GGML_ROPE_TYPE_NEOX 2
+#define GGML_ROPE_TYPE_MROPE 8
+#define GGML_ROPE_TYPE_VISION 24
+#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000
+
+#define GGML_MROPE_SECTIONS 4
+
+#define GGML_UNUSED(x) (void)(x)
+#ifdef __CUDACC__
+template<typename... Args>
+__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {}
+#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__)
+#else
+#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0)
+#endif // __CUDACC__
+
+#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
+
+#ifndef NDEBUG
+# define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0)
+#elif defined(__GNUC__)
+# define GGML_UNREACHABLE() __builtin_unreachable()
+#elif defined(_MSC_VER)
+# define GGML_UNREACHABLE() __assume(0)
+#else
+# define GGML_UNREACHABLE() ((void) 0)
+#endif
+
+#ifdef __cplusplus
+# define GGML_NORETURN [[noreturn]]
+#elif defined(_MSC_VER)
+# define GGML_NORETURN __declspec(noreturn)
+#else
+# define GGML_NORETURN _Noreturn
+#endif
+
+#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__)
+#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x)
+
+// used to copy the number of elements and stride in bytes of tensors into local variables.
+// main purpose is to reduce code duplication and improve readability.
+//
+// example:
+//
+// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
+// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);
+//
+#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
+ const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \
+ GGML_UNUSED(prefix##0);
+#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
+ GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \
+ const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \
+ GGML_UNUSED(prefix##1);
+#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
+ GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \
+ const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \
+ GGML_UNUSED(prefix##2);
+#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
+ GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \
+ const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \
+ GGML_UNUSED(prefix##3);
+
+#define GGML_TENSOR_UNARY_OP_LOCALS \
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+#define GGML_TENSOR_BINARY_OP_LOCALS \
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+#define GGML_TENSOR_TERNARY_OP_LOCALS \
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+#define GGML_TENSOR_BINARY_OP_LOCALS01 \
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ // Function type used in fatal error callbacks
+ typedef void (*ggml_abort_callback_t)(const char * error_message);
+
+ // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)
+ // Returns the old callback for chaining
+ GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback);
+
+ GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)
+ GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);
+
+ enum ggml_status {
+ GGML_STATUS_ALLOC_FAILED = -2,
+ GGML_STATUS_FAILED = -1,
+ GGML_STATUS_SUCCESS = 0,
+ GGML_STATUS_ABORTED = 1,
+ };
+
+ // get ggml_status name string
+ GGML_API const char * ggml_status_to_string(enum ggml_status status);
+
+ // ieee 754-2008 half-precision float16
+ // todo: make this not an integral type
+ typedef uint16_t ggml_fp16_t;
+ GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);
+ GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);
+ GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);
+ GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);
+
+ // google brain half-precision bfloat16
+ typedef struct { uint16_t bits; } ggml_bf16_t;
+ GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
+ GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16
+ GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
+ GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);
+ GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
+
+ struct ggml_object;
+ struct ggml_context;
+ struct ggml_cgraph;
+
+ // NOTE: always add types at the end of the enum to keep backward compatibility
+ enum ggml_type {
+ GGML_TYPE_F32 = 0,
+ GGML_TYPE_F16 = 1,
+ GGML_TYPE_Q4_0 = 2,
+ GGML_TYPE_Q4_1 = 3,
+ // GGML_TYPE_Q4_2 = 4, support has been removed
+ // GGML_TYPE_Q4_3 = 5, support has been removed
+ GGML_TYPE_Q5_0 = 6,
+ GGML_TYPE_Q5_1 = 7,
+ GGML_TYPE_Q8_0 = 8,
+ GGML_TYPE_Q8_1 = 9,
+ GGML_TYPE_Q2_K = 10,
+ GGML_TYPE_Q3_K = 11,
+ GGML_TYPE_Q4_K = 12,
+ GGML_TYPE_Q5_K = 13,
+ GGML_TYPE_Q6_K = 14,
+ GGML_TYPE_Q8_K = 15,
+ GGML_TYPE_IQ2_XXS = 16,
+ GGML_TYPE_IQ2_XS = 17,
+ GGML_TYPE_IQ3_XXS = 18,
+ GGML_TYPE_IQ1_S = 19,
+ GGML_TYPE_IQ4_NL = 20,
+ GGML_TYPE_IQ3_S = 21,
+ GGML_TYPE_IQ2_S = 22,
+ GGML_TYPE_IQ4_XS = 23,
+ GGML_TYPE_I8 = 24,
+ GGML_TYPE_I16 = 25,
+ GGML_TYPE_I32 = 26,
+ GGML_TYPE_I64 = 27,
+ GGML_TYPE_F64 = 28,
+ GGML_TYPE_IQ1_M = 29,
+ GGML_TYPE_BF16 = 30,
+ // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
+ // GGML_TYPE_Q4_0_4_8 = 32,
+ // GGML_TYPE_Q4_0_8_8 = 33,
+ GGML_TYPE_TQ1_0 = 34,
+ GGML_TYPE_TQ2_0 = 35,
+ // GGML_TYPE_IQ4_NL_4_4 = 36,
+ // GGML_TYPE_IQ4_NL_4_8 = 37,
+ // GGML_TYPE_IQ4_NL_8_8 = 38,
+ GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
+ GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
+ GGML_TYPE_Q1_0 = 41,
+ GGML_TYPE_Q2_0 = 42,
+ GGML_TYPE_COUNT = 43,
+ };
+
+ // [TAG_GGML_PREC]
+ // this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op
+ // the declared types can be:
+ // - result accumulation type
+ // - source tensor data representation type
+ // - etc.
+ // the precision parameters are stored as ggml_tensor.op_params to the respective ops
+ enum ggml_prec {
+ GGML_PREC_UNDEFINED = 0,
+ GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED
+ GGML_PREC_F32 = 10,
+ GGML_PREC_BF16 = 15,
+ GGML_PREC_F16 = 20,
+ GGML_PREC_Q8 = 30,
+ GGML_PREC_Q4 = 40,
+ };
+
+ // op hint
+ enum ggml_op_hint {
+ GGML_HINT_NONE = 0,
+ GGML_HINT_SRC0_IS_HADAMARD = 1,
+ };
+
+ // model file types
+ enum ggml_ftype {
+ GGML_FTYPE_UNKNOWN = -1,
+ GGML_FTYPE_ALL_F32 = 0,
+ GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
+ GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
+ GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
+ GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors
+ GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors
+ GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors
+ };
+
+ // available tensor operations:
+ enum ggml_op {
+ GGML_OP_NONE = 0,
+
+ GGML_OP_DUP,
+ GGML_OP_ADD,
+ GGML_OP_ADD_ID,
+ GGML_OP_ADD1,
+ GGML_OP_ACC,
+ GGML_OP_SUB,
+ GGML_OP_MUL,
+ GGML_OP_DIV,
+ GGML_OP_SQR,
+ GGML_OP_SQRT,
+ GGML_OP_LOG,
+ GGML_OP_SIN,
+ GGML_OP_COS,
+ GGML_OP_SUM,
+ GGML_OP_SUM_ROWS,
+ GGML_OP_CUMSUM,
+ GGML_OP_MEAN,
+ GGML_OP_ARGMAX,
+ GGML_OP_COUNT_EQUAL,
+ GGML_OP_REPEAT,
+ GGML_OP_REPEAT_BACK,
+ GGML_OP_CONCAT,
+ GGML_OP_SILU_BACK,
+ GGML_OP_NORM, // normalize
+ GGML_OP_RMS_NORM,
+ GGML_OP_RMS_NORM_BACK,
+ GGML_OP_GROUP_NORM,
+ GGML_OP_L2_NORM,
+
+ GGML_OP_MUL_MAT,
+ GGML_OP_MUL_MAT_ID,
+ GGML_OP_OUT_PROD,
+
+ GGML_OP_SCALE,
+ GGML_OP_SET,
+ GGML_OP_CPY,
+ GGML_OP_CONT,
+ GGML_OP_RESHAPE,
+ GGML_OP_VIEW,
+ GGML_OP_PERMUTE,
+ GGML_OP_TRANSPOSE,
+ GGML_OP_GET_ROWS,
+ GGML_OP_GET_ROWS_BACK,
+ GGML_OP_SET_ROWS,
+ GGML_OP_DIAG,
+ GGML_OP_DIAG_MASK_INF,
+ GGML_OP_DIAG_MASK_ZERO,
+ GGML_OP_SOFT_MAX,
+ GGML_OP_SOFT_MAX_BACK,
+ GGML_OP_ROPE,
+ GGML_OP_ROPE_BACK,
+ GGML_OP_CLAMP,
+ GGML_OP_CONV_TRANSPOSE_1D,
+ GGML_OP_IM2COL,
+ GGML_OP_IM2COL_BACK,
+ GGML_OP_IM2COL_3D,
+ GGML_OP_COL2IM_1D,
+ GGML_OP_CONV_2D,
+ GGML_OP_CONV_3D,
+ GGML_OP_CONV_2D_DW,
+ GGML_OP_CONV_TRANSPOSE_2D,
+ GGML_OP_POOL_1D,
+ GGML_OP_POOL_2D,
+ GGML_OP_POOL_2D_BACK,
+ GGML_OP_UPSCALE,
+ GGML_OP_PAD,
+ GGML_OP_PAD_REFLECT_1D,
+ GGML_OP_ROLL,
+ GGML_OP_ARANGE,
+ GGML_OP_TIMESTEP_EMBEDDING,
+ GGML_OP_ARGSORT,
+ GGML_OP_TOP_K,
+ GGML_OP_LEAKY_RELU,
+ GGML_OP_TRI,
+ GGML_OP_FILL,
+
+ GGML_OP_FLASH_ATTN_EXT,
+ GGML_OP_FLASH_ATTN_BACK,
+ GGML_OP_SSM_CONV,
+ GGML_OP_SSM_SCAN,
+ GGML_OP_WIN_PART,
+ GGML_OP_WIN_UNPART,
+ GGML_OP_GET_REL_POS,
+ GGML_OP_ADD_REL_POS,
+ GGML_OP_RWKV_WKV6,
+ GGML_OP_GATED_LINEAR_ATTN,
+ GGML_OP_RWKV_WKV7,
+ GGML_OP_SOLVE_TRI,
+ GGML_OP_GATED_DELTA_NET,
+ GGML_OP_LIGHTNING_INDEXER,
+ GGML_OP_DSV4_HC_COMB,
+ GGML_OP_DSV4_HC_PRE,
+ GGML_OP_DSV4_HC_POST,
+
+ GGML_OP_UNARY,
+
+ GGML_OP_MAP_CUSTOM1,
+ GGML_OP_MAP_CUSTOM2,
+ GGML_OP_MAP_CUSTOM3,
+
+ GGML_OP_CUSTOM,
+
+ GGML_OP_CROSS_ENTROPY_LOSS,
+ GGML_OP_CROSS_ENTROPY_LOSS_BACK,
+ GGML_OP_OPT_STEP_ADAMW,
+ GGML_OP_OPT_STEP_SGD,
+
+ GGML_OP_GLU,
+
+ GGML_OP_COUNT,
+ };
+
+ enum ggml_unary_op {
+ GGML_UNARY_OP_ABS,
+ GGML_UNARY_OP_SGN,
+ GGML_UNARY_OP_NEG,
+ GGML_UNARY_OP_STEP,
+ GGML_UNARY_OP_TANH,
+ GGML_UNARY_OP_ELU,
+ GGML_UNARY_OP_RELU,
+ GGML_UNARY_OP_SIGMOID,
+ GGML_UNARY_OP_GELU,
+ GGML_UNARY_OP_GELU_QUICK,
+ GGML_UNARY_OP_SILU,
+ GGML_UNARY_OP_HARDSWISH,
+ GGML_UNARY_OP_HARDSIGMOID,
+ GGML_UNARY_OP_EXP,
+ GGML_UNARY_OP_EXPM1,
+ GGML_UNARY_OP_SOFTPLUS,
+ GGML_UNARY_OP_GELU_ERF,
+ GGML_UNARY_OP_XIELU,
+ GGML_UNARY_OP_FLOOR,
+ GGML_UNARY_OP_CEIL,
+ GGML_UNARY_OP_ROUND,
+ GGML_UNARY_OP_TRUNC,
+
+ GGML_UNARY_OP_COUNT,
+ };
+
+ enum ggml_glu_op {
+ GGML_GLU_OP_REGLU,
+ GGML_GLU_OP_GEGLU,
+ GGML_GLU_OP_SWIGLU,
+ GGML_GLU_OP_SWIGLU_OAI,
+ GGML_GLU_OP_GEGLU_ERF,
+ GGML_GLU_OP_GEGLU_QUICK,
+ GGML_GLU_OP_SWIGLU_CLAMP,
+
+ GGML_GLU_OP_COUNT,
+ };
+
+ enum ggml_object_type {
+ GGML_OBJECT_TYPE_TENSOR,
+ GGML_OBJECT_TYPE_GRAPH,
+ GGML_OBJECT_TYPE_WORK_BUFFER
+ };
+
+ enum ggml_log_level {
+ GGML_LOG_LEVEL_NONE = 0,
+ GGML_LOG_LEVEL_DEBUG = 1,
+ GGML_LOG_LEVEL_INFO = 2,
+ GGML_LOG_LEVEL_WARN = 3,
+ GGML_LOG_LEVEL_ERROR = 4,
+ GGML_LOG_LEVEL_CONT = 5, // continue previous log
+ };
+
+ // this tensor...
+ enum ggml_tensor_flag {
+ GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph
+ GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph
+ GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters
+ GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up)
+ GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed
+ };
+
+ enum ggml_tri_type {
+ GGML_TRI_TYPE_UPPER_DIAG = 0,
+ GGML_TRI_TYPE_UPPER = 1,
+ GGML_TRI_TYPE_LOWER_DIAG = 2,
+ GGML_TRI_TYPE_LOWER = 3
+ };
+
+ struct ggml_init_params {
+ // memory pool
+ size_t mem_size; // bytes
+ void * mem_buffer; // if NULL, memory will be allocated internally
+ bool no_alloc; // don't allocate memory for the tensor data
+ };
+
+ // n-dimensional tensor
+ struct ggml_tensor {
+ enum ggml_type type;
+
+ struct ggml_backend_buffer * buffer;
+
+ int64_t ne[GGML_MAX_DIMS]; // number of elements
+ size_t nb[GGML_MAX_DIMS]; // stride in bytes:
+ // nb[0] = ggml_type_size(type)
+ // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding
+ // nb[i] = nb[i-1] * ne[i-1]
+
+ // compute data
+ enum ggml_op op;
+
+ // op params - allocated as int32_t for alignment
+ int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
+
+ int32_t flags;
+
+ struct ggml_tensor * src[GGML_MAX_SRC];
+
+ // source tensor and offset for views
+ struct ggml_tensor * view_src;
+ size_t view_offs;
+
+ void * data;
+
+ char name[GGML_MAX_NAME];
+
+ void * extra; // extra things e.g. for ggml-cuda.cu
+
+ char padding[8];
+ };
+
+ static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
+
+ // Abort callback
+ // If not NULL, called before ggml computation
+ // If it returns true, the computation is aborted
+ typedef bool (*ggml_abort_callback)(void * data);
+
+
+ //
+ // GUID
+ //
+
+ // GUID types
+ typedef uint8_t ggml_guid[16];
+ typedef ggml_guid * ggml_guid_t;
+
+ GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);
+
+ // misc
+
+ GGML_API const char * ggml_version(void);
+ GGML_API const char * ggml_commit(void);
+
+ GGML_API void ggml_time_init(void); // call this once at the beginning of the program
+ GGML_API int64_t ggml_time_ms(void);
+ GGML_API int64_t ggml_time_us(void);
+ GGML_API int64_t ggml_cycles(void);
+ GGML_API int64_t ggml_cycles_per_ms(void);
+
+ // accepts a UTF-8 path, even on Windows
+ GGML_API FILE * ggml_fopen(const char * fname, const char * mode);
+
+ GGML_API void ggml_print_object (const struct ggml_object * obj);
+ GGML_API void ggml_print_objects(const struct ggml_context * ctx);
+
+ GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);
+ GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor);
+ GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor);
+ GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN
+
+ GGML_API int64_t ggml_blck_size(enum ggml_type type);
+ GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block
+ GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row
+
+ GGML_DEPRECATED(
+ GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float
+ "use ggml_row_size() instead");
+
+ GGML_API const char * ggml_type_name(enum ggml_type type);
+ GGML_API const char * ggml_op_name (enum ggml_op op);
+ GGML_API const char * ggml_op_symbol(enum ggml_op op);
+
+ GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);
+ GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op);
+ GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name
+
+ GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);
+
+ GGML_API bool ggml_is_quantized(enum ggml_type type);
+
+ // TODO: temporary until model loading of ggml examples is refactored
+ GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);
+
+ GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_view (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);
+ GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars
+
+ // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation)
+ GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor);
+ GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()
+ GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1
+ GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2
+
+ GGML_API bool ggml_is_contiguous_to_1(const struct ggml_tensor * tensor); // contiguous for dims < 1
+ GGML_API bool ggml_is_contiguous_to_2(const struct ggml_tensor * tensor); // contiguous for dims < 2
+ GGML_API bool ggml_is_contiguous_to_3(const struct ggml_tensor * tensor); // contiguous for dims < 3
+
+ // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok)
+ GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor);
+
+ // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN
+ GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor);
+
+ // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements
+ GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor);
+
+ GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);
+ GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
+
+ GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
+
+ // use this to compute the memory overhead of a tensor
+ GGML_API size_t ggml_tensor_overhead(void);
+
+ GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);
+
+ // main
+
+ GGML_API struct ggml_context * ggml_init (struct ggml_init_params params);
+ GGML_API void ggml_reset(struct ggml_context * ctx);
+ GGML_API void ggml_free (struct ggml_context * ctx);
+
+ GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);
+
+ GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);
+ GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);
+
+ GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);
+ GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);
+ GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int n_dims,
+ const int64_t *ne);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor_1d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor_2d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor_3d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2);
+
+ GGML_API struct ggml_tensor * ggml_new_tensor_4d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3);
+
+ GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes);
+
+ GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);
+ GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);
+
+ // Context tensor enumeration and lookup
+ GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);
+ GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);
+ GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);
+
+ // Converts a flat index into coordinates
+ GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
+
+ GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);
+ GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor);
+
+ GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);
+ GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
+
+ GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);
+ GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);
+ GGML_ATTRIBUTE_FORMAT(2, 3)
+ GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);
+
+ // Tensor flags
+ GGML_API void ggml_set_input(struct ggml_tensor * tensor);
+ GGML_API void ggml_set_output(struct ggml_tensor * tensor);
+ GGML_API void ggml_set_param(struct ggml_tensor * tensor);
+ GGML_API void ggml_set_loss(struct ggml_tensor * tensor);
+
+ //
+ // operations on tensors with backpropagation
+ //
+
+ GGML_API struct ggml_tensor * ggml_dup(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_dup_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_add(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_add_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_add_cast(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ enum ggml_type type);
+
+ // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]]
+ GGML_API struct ggml_tensor * ggml_add_id(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * ids);
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b),
+ "use ggml_add instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b),
+ "use ggml_add_inplace instead");
+
+ // dst = a
+ // view(dst, nb1, nb2, nb3, offset) += b
+ // return dst
+ GGML_API struct ggml_tensor * ggml_acc(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_acc_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_sub(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_sub_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_mul(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_mul_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_div(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_div_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_sqr(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sqr_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sqrt(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sqrt_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_log(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_log_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_expm1(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_expm1_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_softplus(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_softplus_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sin(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sin_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_cos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_cos_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // return scalar
+ GGML_API struct ggml_tensor * ggml_sum(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
+ GGML_API struct ggml_tensor * ggml_sum_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_cumsum(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // mean along rows
+ GGML_API struct ggml_tensor * ggml_mean(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // argmax along rows
+ GGML_API struct ggml_tensor * ggml_argmax(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // count number of equal elements in a and b
+ GGML_API struct ggml_tensor * ggml_count_equal(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // if a is the same shape as b, and a is not parameter, return a
+ // otherwise, return a new tensor: repeat(a) to fit in b
+ GGML_API struct ggml_tensor * ggml_repeat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // repeat a to the specified shape
+ GGML_API struct ggml_tensor * ggml_repeat_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3);
+
+ // sums repetitions in a into shape of b
+ GGML_API struct ggml_tensor * ggml_repeat_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride
+
+ // concat a and b along dim
+ // used in stable-diffusion
+ GGML_API struct ggml_tensor * ggml_concat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int dim);
+
+ GGML_API struct ggml_tensor * ggml_abs(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_abs_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sgn(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sgn_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_neg(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_neg_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_step(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_step_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_tanh(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_tanh_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_elu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_elu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_relu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_leaky_relu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, float negative_slope, bool inplace);
+
+ GGML_API struct ggml_tensor * ggml_relu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sigmoid(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_sigmoid_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // GELU using erf (error function) when possible
+ // some backends may fallback to approximation based on Abramowitz and Stegun formula
+ GGML_API struct ggml_tensor * ggml_gelu_erf(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu_erf_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu_quick(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_silu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_silu_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // a - dy
+ // b - x
+ GGML_API struct ggml_tensor * ggml_silu_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // hardswish(x) = x * relu6(x + 3) / 6
+ GGML_API struct ggml_tensor * ggml_hardswish(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // hardsigmoid(x) = relu6(x + 3) / 6
+ GGML_API struct ggml_tensor * ggml_hardsigmoid(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_exp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_exp_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_floor(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_floor_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_ceil(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_ceil_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_round(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_round_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ /**
+ * Truncates the fractional part of each element in the tensor (towards zero).
+ * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0
+ * Similar to std::trunc in C/C++.
+ */
+
+ GGML_API struct ggml_tensor * ggml_trunc(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_trunc_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+
+
+ // xIELU activation function
+ // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0)
+ // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions
+ // that constrain the positive and negative source alpha values respectively
+ GGML_API struct ggml_tensor * ggml_xielu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float alpha_n,
+ float alpha_p,
+ float beta,
+ float eps);
+
+ // gated linear unit ops
+ // A: n columns, r rows,
+ // result is n / 2 columns, r rows,
+ // expects gate in second half of row, unless swapped is true
+ GGML_API struct ggml_tensor * ggml_glu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_glu_op op,
+ bool swapped);
+
+ GGML_API struct ggml_tensor * ggml_reglu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_reglu_swapped(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_geglu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_geglu_swapped(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_swiglu(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_swiglu_swapped(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_geglu_erf(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_geglu_erf_swapped(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_geglu_quick(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ GGML_API struct ggml_tensor * ggml_geglu_quick_swapped(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // A: n columns, r rows,
+ // B: n columns, r rows,
+ GGML_API struct ggml_tensor * ggml_glu_split(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ enum ggml_glu_op op);
+
+ GGML_API struct ggml_tensor * ggml_reglu_split(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_geglu_split(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_swiglu_split(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_geglu_erf_split(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_geglu_quick_split(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ GGML_API struct ggml_tensor * ggml_swiglu_oai(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float alpha,
+ float limit);
+
+ GGML_API struct ggml_tensor * ggml_swiglu_clamp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float limit);
+
+ // normalize along rows
+ GGML_API struct ggml_tensor * ggml_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_rms_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_rms_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ // group normalize along ne0*ne1*n_groups
+ // used in stable-diffusion
+ GGML_API struct ggml_tensor * ggml_group_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_groups,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_group_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_groups,
+ float eps);
+
+ // l2 normalize along rows
+ // used in rwkv v7
+ GGML_API struct ggml_tensor * ggml_l2_norm(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ GGML_API struct ggml_tensor * ggml_l2_norm_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float eps);
+
+ // a - x
+ // b - dy
+ GGML_API struct ggml_tensor * ggml_rms_norm_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float eps);
+
+ // [TAG_GGML_PREC]
+ // set the minimum required accumulator type for the implementation to use during the compute
+ // for example:
+ // - GGML_PREC_F32 - requires accumulation of the results in F32
+ // - GGML_PREC_BF16 - can accumulate the results in BF16, F32
+ // - GGML_PREC_F16 - can accumulate the results in F16, F32
+ // - GGML_PREC_Q8 - not allowed
+ // - GGML_PREC_Q4 - not allowed
+ //
+ // return false on faliure
+ GGML_API bool ggml_prec_set_acc(
+ struct ggml_tensor * a,
+ enum ggml_prec prec);
+
+ // [TAG_GGML_PREC]
+ // set the smallest rank that the implementation can use to internally convert the src[idx] data to
+ // ranks in decreasing order:
+ // - GGML_PREC_F32 - GGML_TYPE_F32
+ // - GGML_PREC_BF16 - GGML_TYPE_BF16
+ // - GGML_PREC_F16 - GGML_TYPE_F16,
+ // - GGML_PREC_Q8 - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc.
+ // - GGML_PREC_Q4 - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc.
+ //
+ // for example:
+ // - ggml_prec_set_src(a, GGML_PREC_Q8, 1):
+ // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0
+ // - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4
+ // - ggml_prec_set_src(a, GGML_PREC_Q4, 1):
+ // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc.
+ //
+ // return false on faliure
+ GGML_API bool ggml_prec_set_src(
+ struct ggml_tensor * a,
+ enum ggml_prec prec,
+ int idx);
+
+ // A: k columns, n rows => [ne03, ne02, n, k]
+ // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
+ // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
+ GGML_API struct ggml_tensor * ggml_mul_mat(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // change the precision of a matrix multiplication
+ // set to GGML_PREC_F32 for higher precision (useful for phi-2)
+ GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec(
+ struct ggml_tensor * a,
+ enum ggml_prec prec),
+ "use ggml_prec_set_acc() instead");
+
+ // change the hint of a matrix multiplication
+ GGML_API void ggml_mul_mat_set_hint(
+ struct ggml_tensor * a,
+ enum ggml_op_hint hint);
+
+ // indirect matrix multiplication
+ //
+ // [TAG_MUL_MAT_ID_DUPLICATE_ID]
+ // note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`
+ // (i.e. duplicate experts per token are not guaranteed to be supported in all backends)
+ // TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533
+ GGML_API struct ggml_tensor * ggml_mul_mat_id(
+ struct ggml_context * ctx,
+ struct ggml_tensor * as,
+ struct ggml_tensor * b,
+ struct ggml_tensor * ids);
+
+ // A: m columns, n rows,
+ // B: p columns, n rows,
+ // result is m columns, p rows
+ GGML_API struct ggml_tensor * ggml_out_prod(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ //
+ // operations on tensors without backpropagation
+ //
+
+ GGML_API struct ggml_tensor * ggml_scale(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_scale_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s);
+
+ // x = s * a + b
+ GGML_API struct ggml_tensor * ggml_scale_bias(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s,
+ float b);
+
+ GGML_API struct ggml_tensor * ggml_scale_bias_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float s,
+ float b);
+
+ // b -> view(a,offset,nb1,nb2,3), return modified a
+ GGML_API struct ggml_tensor * ggml_set(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset); // in bytes
+
+ // b -> view(a,offset,nb1,nb2,3), return view(a)
+ GGML_API struct ggml_tensor * ggml_set_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t nb2,
+ size_t nb3,
+ size_t offset); // in bytes
+
+ GGML_API struct ggml_tensor * ggml_set_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t offset); // in bytes
+
+ GGML_API struct ggml_tensor * ggml_set_1d_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t offset); // in bytes
+
+ // b -> view(a,offset,nb1,nb2,3), return modified a
+ GGML_API struct ggml_tensor * ggml_set_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t offset); // in bytes
+
+ // b -> view(a,offset,nb1,nb2,3), return view(a)
+ GGML_API struct ggml_tensor * ggml_set_2d_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ size_t nb1,
+ size_t offset); // in bytes
+
+ // a -> b, return view(b)
+ GGML_API struct ggml_tensor * ggml_cpy(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // note: casting from f32 to i32 will discard the fractional part
+ GGML_API struct ggml_tensor * ggml_cast(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_type type);
+
+ // make contiguous
+ GGML_API struct ggml_tensor * ggml_cont(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // make contiguous, with new shape
+ GGML_API struct ggml_tensor * ggml_cont_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0);
+
+ GGML_API struct ggml_tensor * ggml_cont_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1);
+
+ GGML_API struct ggml_tensor * ggml_cont_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2);
+
+ GGML_API struct ggml_tensor * ggml_cont_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3);
+
+ // return view(a), b specifies the new shape
+ // TODO: when we start computing gradient, make a copy instead of view
+ GGML_API struct ggml_tensor * ggml_reshape(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // return view(a)
+ // TODO: when we start computing gradient, make a copy instead of view
+ GGML_API struct ggml_tensor * ggml_reshape_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0);
+
+ GGML_API struct ggml_tensor * ggml_reshape_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1);
+
+ // return view(a)
+ // TODO: when we start computing gradient, make a copy instead of view
+ GGML_API struct ggml_tensor * ggml_reshape_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2);
+
+ GGML_API struct ggml_tensor * ggml_reshape_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3);
+
+ // offset in bytes
+ GGML_API struct ggml_tensor * ggml_view_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_view_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ size_t nb1, // row stride in bytes
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_view_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ size_t nb1, // row stride in bytes
+ size_t nb2, // slice stride in bytes
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_view_4d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3,
+ size_t nb1, // row stride in bytes
+ size_t nb2, // slice stride in bytes
+ size_t nb3,
+ size_t offset);
+
+ GGML_API struct ggml_tensor * ggml_permute(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int axis0,
+ int axis1,
+ int axis2,
+ int axis3);
+
+ // alias for ggml_permute(ctx, a, 1, 0, 2, 3)
+ GGML_API struct ggml_tensor * ggml_transpose(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // supports 4D a:
+ // a [n_embd, ne1, ne2, ne3]
+ // b I32 [n_rows, ne2, ne3, 1]
+ //
+ // return [n_embd, n_rows, ne2, ne3]
+ GGML_API struct ggml_tensor * ggml_get_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // data
+ struct ggml_tensor * b); // row indices
+
+ GGML_API struct ggml_tensor * ggml_get_rows_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // gradients of ggml_get_rows result
+ struct ggml_tensor * b, // row indices
+ struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape
+
+ // a TD [n_embd, ne1, ne2, ne3]
+ // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3
+ // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1)
+ //
+ // undefined behavior if destination rows overlap
+ //
+ // broadcast:
+ // ne2 % ne11 == 0
+ // ne3 % ne12 == 0
+ //
+ // return view(a)
+ GGML_API struct ggml_tensor * ggml_set_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // destination
+ struct ggml_tensor * b, // source
+ struct ggml_tensor * c); // row indices
+
+ GGML_API struct ggml_tensor * ggml_diag(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // set elements above the diagonal to -INF
+ GGML_API struct ggml_tensor * ggml_diag_mask_inf(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past);
+
+ // set elements above the diagonal to 0
+ GGML_API struct ggml_tensor * ggml_diag_mask_zero(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int n_past);
+
+ GGML_API struct ggml_tensor * ggml_clamp(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float min,
+ float max);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_clamp_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float min,
+ float max);
+
+ GGML_API struct ggml_tensor * ggml_soft_max(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_soft_max_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a);
+
+ // a [ne0, ne01, ne02, ne03]
+ // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional
+ //
+ // broadcast:
+ // ne02 % ne12 == 0
+ // ne03 % ne13 == 0
+ //
+ // fused soft_max(a*scale + mask*(ALiBi slope))
+ // max_bias = 0.0f for no ALiBi
+ GGML_API struct ggml_tensor * ggml_soft_max_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * mask,
+ float scale,
+ float max_bias);
+
+ GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * mask,
+ float scale,
+ float max_bias);
+
+ GGML_API void ggml_soft_max_add_sinks(
+ struct ggml_tensor * a,
+ struct ggml_tensor * sinks);
+
+ GGML_API struct ggml_tensor * ggml_soft_max_ext_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float scale,
+ float max_bias);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ float scale,
+ float max_bias);
+
+ // rotary position embedding
+ // if (mode & 1) - skip n_past elements (NOT SUPPORTED)
+ // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style
+ //
+ // b is an int32 vector with size a->ne[2], it contains the positions
+ GGML_API struct ggml_tensor * ggml_rope(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_rope_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode);
+
+ // RoPE operations with extended options
+ // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token]
+ // b is an int32 vector with size n_token
+ // c is freq factors (e.g. phi3-128k), (optional)
+ // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi
+ //
+ // pseudo-code for computing theta:
+ // for i in [0, n_dims/2):
+ // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims);
+ // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c
+ // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here
+ //
+ // other params are used by YaRN RoPE scaling, these default values will disable YaRN:
+ // freq_scale = 1.0f
+ // ext_factor = 0.0f
+ // attn_factor = 1.0f
+ // beta_fast = 0.0f
+ // beta_slow = 0.0f
+ //
+ // example:
+ // (marking: c = cos, s = sin, 0 = unrotated)
+ // given a single head with size = 8 --> [00000000]
+ // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000]
+ // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs]
+ // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000]
+ // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss]
+ GGML_API struct ggml_tensor * ggml_rope_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ // multi-dimensional RoPE, for Qwen-VL and similar vision models
+ // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX
+ // sections specify how many dimensions to rotate in each section:
+ // section length is equivalent to number of cos/sin pairs, NOT the number of dims
+ // (i.e. sum of 4 sections are expected to be n_dims/2)
+ // last sections can be 0, means ignored
+ // all other options are identical to ggml_rope_ext
+ //
+ // important note:
+ // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION
+ // if you need normal ordering, there are 2 methods:
+ // (1) split the tensor manually using ggml_view
+ // (2) permute the weight upon conversion
+ // - for VISION, n_dims must be head_size/2
+ //
+ // example M-RoPE:
+ // given sections = [t=4, y=2, x=2, 0]
+ // given a single head with size = 18 --> [000000000000000000]
+ // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering)
+ // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering)
+ // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section
+ // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section
+ //
+ // example vision RoPE:
+ // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored)
+ // given a single head with size = 8 --> [00000000]
+ // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx]
+ // other values of n_dims are untested and is undefined behavior
+ // note: unlike MROPE, the theta for each dim is computed differently for each section
+ // in other words, idx used for theta: [0123] for y section, then [0123] for x section
+ GGML_API struct ggml_tensor * ggml_rope_multi(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int sections[GGML_MROPE_SECTIONS],
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ // in-place, returns view(a)
+ GGML_API struct ggml_tensor * ggml_rope_ext_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ GGML_API struct ggml_tensor * ggml_rope_multi_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int sections[GGML_MROPE_SECTIONS],
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow),
+ "use ggml_rope_ext instead");
+
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow),
+ "use ggml_rope_ext_inplace instead");
+
+ // compute correction dims for YaRN RoPE scaling
+ GGML_API void ggml_rope_yarn_corr_dims(
+ int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
+
+ // rotary position embedding backward, i.e compute dx from dy
+ // a - dy
+ GGML_API struct ggml_tensor * ggml_rope_ext_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // gradients of ggml_rope result
+ struct ggml_tensor * b, // positions
+ struct ggml_tensor * c, // freq factors
+ int n_dims,
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ GGML_API struct ggml_tensor * ggml_rope_multi_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ int n_dims,
+ int sections[4],
+ int mode,
+ int n_ctx_orig,
+ float freq_base,
+ float freq_scale,
+ float ext_factor,
+ float attn_factor,
+ float beta_fast,
+ float beta_slow);
+
+ // set the offset dims for RoPE
+ // a must be GGML_OP_ROPE or GGML_OP_ROPE_BACK
+ // vision RoPE is not supported
+ // example: (marking: x = rotated, 0 = unrotated)
+ // n_embd = 10, n_dims = 4, offset = 2 --> [00xxxx0000]
+ GGML_API struct ggml_tensor * ggml_rope_set_offset(
+ struct ggml_tensor * a,
+ int n_offs);
+
+ // im2col
+ // converts data into a format that effectively results in a convolution when combined with matrix multiplication
+ GGML_API struct ggml_tensor * ggml_im2col(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1, // dilation dimension 1
+ bool is_2D,
+ enum ggml_type dst_type);
+
+ GGML_API struct ggml_tensor * ggml_im2col_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // gradient of im2col output
+ int64_t * ne, // shape of im2col input
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1, // dilation dimension 1
+ bool is_2D);
+
+ // col2im_1d: scatter-add GEMM columns back to 1D signal
+ // a: [K*OC, T_in] (columns from matmul, K = a->ne[0]/OC)
+ // result: [T_out, OC] where T_out = (T_in - 1)*s0 + K - 2*p0
+ GGML_API struct ggml_tensor * ggml_col2im_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // columns [K*OC, T_in]
+ int s0, // stride
+ int oc, // output channels
+ int p0); // padding to crop from both sides
+
+ GGML_API struct ggml_tensor * ggml_conv_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride
+ int p0, // padding
+ int d0); // dilation
+
+ // conv_1d with padding = half
+ // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
+ GGML_API struct ggml_tensor* ggml_conv_1d_ph(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s, // stride
+ int d); // dilation
+
+ // depthwise
+ // TODO: this is very likely wrong for some cases! - needs more testing
+ GGML_API struct ggml_tensor * ggml_conv_1d_dw(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride
+ int p0, // padding
+ int d0); // dilation
+
+ GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride
+ int d0); // dilation
+
+ GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride
+ int p0, // padding
+ int d0); // dilation
+
+ GGML_API struct ggml_tensor * ggml_conv_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1); // dilation dimension 1
+
+ GGML_API struct ggml_tensor * ggml_im2col_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int64_t IC,
+ int s0, // stride width
+ int s1, // stride height
+ int s2, // stride depth
+ int p0, // padding width
+ int p1, // padding height
+ int p2, // padding depth
+ int d0, // dilation width
+ int d1, // dilation height
+ int d2, // dilation depth
+ enum ggml_type dst_type);
+
+ // a: [OC*IC, KD, KH, KW]
+ // b: [N*IC, ID, IH, IW]
+ // result: [N*OC, OD, OH, OW]
+ GGML_API struct ggml_tensor * ggml_conv_3d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int64_t IC,
+ int s0, // stride width
+ int s1, // stride height
+ int s2, // stride depth
+ int p0, // padding width
+ int p1, // padding height
+ int p2, // padding depth
+ int d0, // dilation width
+ int d1, // dilation height
+ int d2 // dilation depth
+ );
+
+ // kernel size is a->ne[0] x a->ne[1]
+ // stride is equal to kernel size
+ // padding is zero
+ // example:
+ // a: 16 16 3 768
+ // b: 1024 1024 3 1
+ // res: 64 64 768 1
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // kernel size is a->ne[0] x a->ne[1]
+ // stride is 1
+ // padding is half
+ // example:
+ // a: 3 3 256 256
+ // b: 64 64 256 1
+ // res: 64 64 256 1
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b);
+
+ // depthwise (via im2col and mul_mat)
+ GGML_API struct ggml_tensor * ggml_conv_2d_dw(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel
+ struct ggml_tensor * b, // data
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1); // dilation dimension 1
+
+ // Depthwise 2D convolution
+ // may be faster than ggml_conv_2d_dw, but not available in all backends
+ // a: KW KH 1 C convolution kernel
+ // b: W H C N input data
+ // res: W_out H_out C N
+ GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int stride0,
+ int stride1,
+ int pad0,
+ int pad1,
+ int dilation0,
+ int dilation1);
+
+ GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ int stride);
+
+ GGML_API struct ggml_tensor * ggml_conv_2d_direct(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC]
+ struct ggml_tensor * b, // input data [W, H, C, N]
+ int s0, // stride dimension 0
+ int s1, // stride dimension 1
+ int p0, // padding dimension 0
+ int p1, // padding dimension 1
+ int d0, // dilation dimension 0
+ int d1); // dilation dimension 1
+
+ GGML_API struct ggml_tensor * ggml_conv_3d_direct(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC]
+ struct ggml_tensor * b, // input [W, H, D, C * N]
+ int s0, // stride
+ int s1,
+ int s2,
+ int p0, // padding
+ int p1,
+ int p2,
+ int d0, // dilation
+ int d1,
+ int d2,
+ int n_channels,
+ int n_batch,
+ int n_channels_out);
+
+ enum ggml_op_pool {
+ GGML_OP_POOL_MAX,
+ GGML_OP_POOL_AVG,
+ GGML_OP_POOL_COUNT,
+ };
+
+ GGML_API struct ggml_tensor * ggml_pool_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_op_pool op,
+ int k0, // kernel size
+ int s0, // stride
+ int p0); // padding
+
+ // the result will have 2*p0 padding for the first dimension
+ // and 2*p1 padding for the second dimension
+ GGML_API struct ggml_tensor * ggml_pool_2d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_op_pool op,
+ int k0,
+ int k1,
+ int s0,
+ int s1,
+ float p0,
+ float p1);
+
+ GGML_API struct ggml_tensor * ggml_pool_2d_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * af, // "a"/input used in forward pass
+ enum ggml_op_pool op,
+ int k0,
+ int k1,
+ int s0,
+ int s1,
+ float p0,
+ float p1);
+
+ enum ggml_scale_mode {
+ GGML_SCALE_MODE_NEAREST = 0,
+ GGML_SCALE_MODE_BILINEAR = 1,
+ GGML_SCALE_MODE_BICUBIC = 2,
+
+ GGML_SCALE_MODE_COUNT
+ };
+
+ enum ggml_scale_flag {
+ GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8),
+ GGML_SCALE_FLAG_ANTIALIAS = (1 << 9),
+ };
+
+ // interpolate
+ // multiplies ne0 and ne1 by scale factor
+ GGML_API struct ggml_tensor * ggml_upscale(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int scale_factor,
+ enum ggml_scale_mode mode);
+
+ // interpolate
+ // interpolate scale to specified dimensions
+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int ne0,
+ int ne1,
+ int ne2,
+ int ne3,
+ enum ggml_scale_mode mode),
+ "use ggml_interpolate instead");
+
+ // Up- or downsamples the input to the specified size.
+ // 2D scale modes (eg. bilinear) are applied to the first two dimensions.
+ GGML_API struct ggml_tensor * ggml_interpolate(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3,
+ uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...]
+
+ // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
+ GGML_API struct ggml_tensor * ggml_pad(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int p0,
+ int p1,
+ int p2,
+ int p3);
+
+ // pad each dimension with values on the other side of the torus (looping around)
+ GGML_API struct ggml_tensor * ggml_pad_circular(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int p0,
+ int p1,
+ int p2,
+ int p3);
+
+ GGML_API struct ggml_tensor * ggml_pad_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int lp0,
+ int rp0,
+ int lp1,
+ int rp1,
+ int lp2,
+ int rp2,
+ int lp3,
+ int rp3
+ );
+
+ // pad each dimension with values on the other side of the torus (looping around)
+ GGML_API struct ggml_tensor * ggml_pad_ext_circular(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int lp0,
+ int rp0,
+ int lp1,
+ int rp1,
+ int lp2,
+ int rp2,
+ int lp3,
+ int rp3);
+
+ // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c]
+ GGML_API struct ggml_tensor * ggml_pad_reflect_1d(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int p0,
+ int p1);
+
+ // Move tensor elements by an offset given for each dimension. Elements that
+ // are shifted beyond the last position are wrapped around to the beginning.
+ GGML_API struct ggml_tensor * ggml_roll(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int shift0,
+ int shift1,
+ int shift2,
+ int shift3);
+
+ // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing
+ // zeroes everywhere outside the masked area
+ GGML_API struct ggml_tensor * ggml_tri(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_tri_type type);
+
+ // Fill tensor a with constant c
+ GGML_API struct ggml_tensor * ggml_fill(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float c);
+
+ GGML_API struct ggml_tensor * ggml_fill_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ float c);
+
+ // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
+ // timesteps: [N,]
+ // return: [N, dim]
+ GGML_API struct ggml_tensor * ggml_timestep_embedding(
+ struct ggml_context * ctx,
+ struct ggml_tensor * timesteps,
+ int dim,
+ int max_period);
+
+ // sort rows
+ enum ggml_sort_order {
+ GGML_SORT_ORDER_ASC,
+ GGML_SORT_ORDER_DESC,
+ };
+
+ GGML_API struct ggml_tensor * ggml_argsort(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_sort_order order);
+
+ // similar to ggml_top_k but implemented as `argsort` + `view`
+ GGML_API struct ggml_tensor * ggml_argsort_top_k(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int k);
+
+ // top k elements per row
+ // note: the resulting top k indices are in no particular order
+ GGML_API struct ggml_tensor * ggml_top_k(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int k);
+
+ GGML_API struct ggml_tensor * ggml_arange(
+ struct ggml_context * ctx,
+ float start,
+ float stop,
+ float step);
+
+ // q: [n_embd_k, n_batch, n_head, ne3 ]
+ // k: [n_embd_k, n_kv, n_head_kv, ne3 ]
+ // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !!
+ // mask: [n_kv, n_batch, ne32, ne33]
+ // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !!
+ //
+ // broadcast:
+ // n_head % n_head_kv == 0
+ // n_head % ne32 == 0
+ // ne3 % ne33 == 0
+ //
+ GGML_API struct ggml_tensor * ggml_flash_attn_ext(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * mask,
+ float scale,
+ float max_bias,
+ float logit_softcap);
+
+ GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(
+ struct ggml_tensor * a,
+ enum ggml_prec prec),
+ "use ggml_prec_set_acc() instead");
+
+ GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(
+ const struct ggml_tensor * a);
+
+ // Use finite mask entries as a sparse K/V set. Set 0 to disable.
+ // n_kv_max must bound the number of finite entries in every mask row.
+ GGML_API void ggml_flash_attn_ext_set_n_kv_max(
+ struct ggml_tensor * a,
+ int32_t n_kv_max);
+
+ GGML_API void ggml_flash_attn_ext_add_sinks(
+ struct ggml_tensor * a,
+ struct ggml_tensor * sinks);
+
+ // TODO: needs to be adapted to ggml_flash_attn_ext
+ GGML_API struct ggml_tensor * ggml_flash_attn_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * d,
+ bool masked);
+
+ GGML_API struct ggml_tensor * ggml_ssm_conv(
+ struct ggml_context * ctx,
+ struct ggml_tensor * sx,
+ struct ggml_tensor * c);
+
+ GGML_API struct ggml_tensor * ggml_ssm_scan(
+ struct ggml_context * ctx,
+ struct ggml_tensor * s,
+ struct ggml_tensor * x,
+ struct ggml_tensor * dt,
+ struct ggml_tensor * A,
+ struct ggml_tensor * B,
+ struct ggml_tensor * C,
+ struct ggml_tensor * ids,
+ int64_t K);
+
+ // partition into non-overlapping windows with padding if needed
+ // example:
+ // a: 768 64 64 1
+ // w: 14
+ // res: 768 14 14 25
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_win_part(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int w);
+
+ // reverse of ggml_win_part
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_win_unpart(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int w0,
+ int h0,
+ int w);
+
+ GGML_API struct ggml_tensor * ggml_unary(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_unary_op op);
+
+ GGML_API struct ggml_tensor * ggml_unary_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ enum ggml_unary_op op);
+
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_get_rel_pos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ int qh,
+ int kh);
+
+ // used in sam
+ GGML_API struct ggml_tensor * ggml_add_rel_pos(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * pw,
+ struct ggml_tensor * ph);
+
+ GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * pw,
+ struct ggml_tensor * ph);
+
+ GGML_API struct ggml_tensor * ggml_rwkv_wkv6(
+ struct ggml_context * ctx,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * r,
+ struct ggml_tensor * tf,
+ struct ggml_tensor * td,
+ struct ggml_tensor * state);
+
+ GGML_API struct ggml_tensor * ggml_gated_linear_attn(
+ struct ggml_context * ctx,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * q,
+ struct ggml_tensor * g,
+ struct ggml_tensor * state,
+ float scale);
+
+ GGML_API struct ggml_tensor * ggml_rwkv_wkv7(
+ struct ggml_context * ctx,
+ struct ggml_tensor * r,
+ struct ggml_tensor * w,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * state);
+
+ /* Solves a specific equation of the form Ax=B, where A is a triangular matrix
+ * without zeroes on the diagonal (i.e. invertible).
+ * B can have any number of columns, but must have the same number of rows as A
+ * If A is [n, n] and B is [n, m], then the result will be [n, m] as well
+ * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases
+ * where n > 100 sparingly, pre-chunk if necessary.
+ *
+ * If left = false, solves xA=B instead
+ * If lower = false, assumes upper triangular instead
+ * If uni = true, assumes diagonal of A to be all ones (will override actual values)
+ *
+ * TODO: currently only lower, right, non-unitriangular variant is implemented
+ */
+ GGML_API struct ggml_tensor * ggml_solve_tri(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ bool left,
+ bool lower,
+ bool uni);
+
+ // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST]
+ // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306
+ //
+ // tensor shapes (S_k == S_v, H_v % H_k == 0):
+ // q, k : [S_k, H_k, n_tokens, n_seqs]
+ // v : [S_v, H_v, n_tokens, n_seqs]
+ // g : [1, H_v, n_tokens, n_seqs] (scalar gate) or [S_v, H_v, n_tokens, n_seqs] (KDA)
+ // beta : [1, H_v, n_tokens, n_seqs]
+ // state : [S_v, S_v, H_v, n_seqs] -- initial recurrent state s0
+ //
+ // the output packs the attention scores [S_v, H_v, n_tokens, n_seqs] followed by K state
+ // snapshots, most-recent first (slot 0 = final state, slot s = state s tokens back). K == 1
+ // keeps only the final state; when n_tokens < K only slots 0..n_tokens-1 are written.
+ GGML_API struct ggml_tensor * ggml_gated_delta_net(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * v,
+ struct ggml_tensor * g,
+ struct ggml_tensor * beta,
+ struct ggml_tensor * state,
+ int64_t K);
+
+ // DSA lightning indexer
+ //
+ // q: [n_embd_idx, n_head_idx, n_batch, ne3 ]
+ // k: [n_embd_idx, 1, n_kv, ne3 ]
+ // weights: [n_head_idx, n_batch, 1, ne3 ] !! prescaled !!
+ // mask: [n_kv, n_batch, 1, ne33] !! f16 !!
+ // res: [n_kv, n_batch, 1, ne3 ]
+ //
+ // broadcast:
+ // ne3 % ne33 == 0
+ //
+ GGML_API struct ggml_tensor * ggml_lightning_indexer(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * weights,
+ struct ggml_tensor * mask);
+
+ // DeepSeek V4 hyper-connections (ref. https://arxiv.org/pdf/2512.24880)
+ // In short these operations are replacements for the original residual connection (x = transformer(x) + x)
+ // using a richer representation through streams.
+ //
+ // hc_comb: mixes [(2 + hc)*hc, n_tokens], scale [3], base [(2 + hc)*hc]
+ // -> [dst_hc, src_hc, n_tokens]
+ // logits[dst, src, t] = mixes[2*hc + dst + hc*src, t]*scale[2]
+ // + base[2*hc + dst + hc*src]
+ // Softmax over dst, add eps, normalize over src, then repeat normalization
+ // over dst followed by src for iterations 1 through n_iter - 1.
+ GGML_API struct ggml_tensor * ggml_dsv4_hc_comb(
+ struct ggml_context * ctx,
+ struct ggml_tensor * mixes,
+ struct ggml_tensor * scale,
+ struct ggml_tensor * base,
+ float eps,
+ int32_t n_iter);
+
+ // hc_pre: x [n_embd, hc, n_tokens], weights [hc, n_tokens] -> [n_embd, n_tokens]
+ // result[i, t] = sum_h x[i, h, t]*weights[h, t]
+ //
+ GGML_API struct ggml_tensor * ggml_dsv4_hc_pre(
+ struct ggml_context * ctx,
+ struct ggml_tensor * x,
+ struct ggml_tensor * weights);
+
+ // hc_pre with a per-element gate (Qwen3.8-Flash-Next): gate [n_embd, hc, n_tokens]
+ // result[i, t] = scale*sum_h x[i, h, t]*sigmoid(gate[i, h, t])
+ //
+ GGML_API struct ggml_tensor * ggml_dsv4_hc_pre_gated(
+ struct ggml_context * ctx,
+ struct ggml_tensor * x,
+ struct ggml_tensor * gate,
+ float scale);
+
+ // hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens],
+ // post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens]
+ // -> [n_embd, hc, n_tokens]
+ // result[i, dst, t] = x[i, t]*post[dst, t]
+ // + sum_src residual[i, src, t]*comb[dst, src, t]
+ // comb == NULL uses the identity: result[i, dst, t] = x[i, t]*post[dst, t] + residual[i, dst, t]
+ //
+ GGML_API struct ggml_tensor * ggml_dsv4_hc_post(
+ struct ggml_context * ctx,
+ struct ggml_tensor * x,
+ struct ggml_tensor * residual,
+ struct ggml_tensor * post,
+ struct ggml_tensor * comb);
+
+ // custom operators
+
+ typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
+ typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);
+ typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);
+
+#define GGML_N_TASKS_MAX (-1)
+ // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks
+
+ GGML_API struct ggml_tensor * ggml_map_custom1(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ ggml_custom1_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom1_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ ggml_custom1_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom2(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ ggml_custom2_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom2_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ ggml_custom2_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom3(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ ggml_custom3_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_map_custom3_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * b,
+ struct ggml_tensor * c,
+ ggml_custom3_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_custom_4d(
+ struct ggml_context * ctx,
+ enum ggml_type type,
+ int64_t ne0,
+ int64_t ne1,
+ int64_t ne2,
+ int64_t ne3,
+ struct ggml_tensor ** args,
+ int n_args,
+ ggml_custom_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ GGML_API struct ggml_tensor * ggml_custom_inplace(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor ** args,
+ int n_args,
+ ggml_custom_op_t fun,
+ int n_tasks,
+ void * userdata);
+
+ // loss function
+
+ GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // logits
+ struct ggml_tensor * b); // labels
+
+ GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // logits
+ struct ggml_tensor * b, // labels
+ struct ggml_tensor * c); // gradients of cross_entropy_loss result
+
+ // AdamW optimizer step
+ // Paper: https://arxiv.org/pdf/1711.05101v3.pdf
+ // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html
+ GGML_API struct ggml_tensor * ggml_opt_step_adamw(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * grad,
+ struct ggml_tensor * m,
+ struct ggml_tensor * v,
+ struct ggml_tensor * adamw_params); // parameters such as the learning rate
+
+ // stochastic gradient descent step (with weight decay)
+ GGML_API struct ggml_tensor * ggml_opt_step_sgd(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a,
+ struct ggml_tensor * grad,
+ struct ggml_tensor * sgd_params); // alpha, weight decay
+
+ // build forward multiple tensors and select one of them for computing
+ // this is useful for creating graphs that have constant topology but compute different things based on the input
+ // ref: https://github.com/ggml-org/llama.cpp/pull/18550
+ //
+ // nodes:
+ // | - build forward into the graph but do not compute
+ // c - build forward into the graph and compute
+ //
+ // | | ... c ... |
+ // | | ... c ... |
+ // | | ... c ... |
+ // [0 1 ... idx ... n-1] <-- ggml_build_forward_select(..., n, idx)
+ // c
+ // c
+ //
+ // example:
+ // struct ggml_tensor * curs[3];
+ //
+ // curs[0] = compute0(...);
+ // curs[1] = compute1(...);
+ // curs[2] = compute2(...);
+ //
+ // int idx = select_branch(some_input);
+ //
+ // struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx);
+ //
+ GGML_API struct ggml_tensor * ggml_build_forward_select(
+ struct ggml_cgraph * cgraph,
+ struct ggml_tensor ** tensors,
+ int n_tensors,
+ int idx);
+
+ GGML_API void ggml_build_forward_expand(
+ struct ggml_cgraph * cgraph,
+ struct ggml_tensor * tensor);
+
+ // add the tensor and its parents to the graph without marking them for compute
+ // the flag is set later, when the tensor is reached from a node that computes
+ GGML_API void ggml_build_forward_order(
+ struct ggml_cgraph * cgraph,
+ struct ggml_tensor * tensor);
+
+ GGML_API void ggml_build_backward_expand(
+ struct ggml_context * ctx, // context for gradient computation
+ struct ggml_cgraph * cgraph,
+ struct ggml_tensor ** grad_accs);
+
+ // graph allocation in a context
+ GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false
+ GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads);
+ GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads);
+ GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);
+ GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1
+ GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);
+
+ GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph);
+ GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i]
+ GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph);
+ GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph);
+
+ GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
+
+ GGML_API size_t ggml_graph_overhead(void);
+ GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);
+
+ GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name);
+ GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);
+ GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);
+
+ // print info and performance information for the graph
+ GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);
+
+ // dump the graph into a file using the dot format
+ GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename);
+
+ // TODO these functions were sandwiched in the old optimization interface, is there a better place for them?
+ typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);
+
+ // Set callback for all future logging events.
+ // If this is not called, or NULL is supplied, everything is output on stderr.
+ GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data);
+ GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data);
+
+ GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
+
+ //
+ // quantization
+ //
+
+ // - ggml_quantize_init can be called multiple times with the same type
+ // it will only initialize the quantization tables for the first call or after ggml_quantize_free
+ // automatically called by ggml_quantize_chunk for convenience
+ //
+ // - ggml_quantize_free will free any memory allocated by ggml_quantize_init
+ // call this at the end of the program to avoid memory leaks
+ //
+ // note: these are thread-safe
+ //
+ GGML_API void ggml_quantize_init(enum ggml_type type);
+ GGML_API void ggml_quantize_free(void);
+
+ // some quantization type cannot be used without an importance matrix
+ GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);
+
+ // calls ggml_quantize_init internally (i.e. can allocate memory)
+ GGML_API size_t ggml_quantize_chunk(
+ enum ggml_type type,
+ const float * src,
+ void * dst,
+ int64_t start,
+ int64_t nrows,
+ int64_t n_per_row,
+ const float * imatrix);
+
+#ifdef __cplusplus
+ // restrict not standard in C++
+# if defined(__GNUC__)
+# define GGML_RESTRICT __restrict__
+# elif defined(__clang__)
+# define GGML_RESTRICT __restrict
+# elif defined(_MSC_VER)
+# define GGML_RESTRICT __restrict
+# else
+# define GGML_RESTRICT
+# endif
+#else
+# if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L)
+# define GGML_RESTRICT __restrict
+# else
+# define GGML_RESTRICT restrict
+# endif
+#endif
+ typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+ typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+ struct ggml_type_traits {
+ const char * type_name;
+ int64_t blck_size;
+ int64_t blck_size_interleave; // interleave elements in blocks
+ size_t type_size;
+ bool is_quantized;
+ ggml_to_float_t to_float;
+ ggml_from_float_t from_float_ref;
+ };
+
+ GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);
+
+ // ggml threadpool
+ // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend
+ // the goal should be to create an API that other backends can use move everything to the ggml base
+
+ // scheduling priorities
+ enum ggml_sched_priority {
+ GGML_SCHED_PRIO_LOW = -1,
+ GGML_SCHED_PRIO_NORMAL,
+ GGML_SCHED_PRIO_MEDIUM,
+ GGML_SCHED_PRIO_HIGH,
+ GGML_SCHED_PRIO_REALTIME
+ };
+
+ // threadpool params
+ // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
+ struct ggml_threadpool_params {
+ bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
+ int n_threads; // number of threads
+ enum ggml_sched_priority prio; // thread priority
+ uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling)
+ bool strict_cpu; // strict cpu placement
+ bool paused; // start in paused state
+ };
+
+ struct ggml_threadpool; // forward declaration, see ggml.c
+
+ typedef struct ggml_threadpool * ggml_threadpool_t;
+
+ GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
+ GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads);
+ GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/python/freetoken/kernel/csrc/ggml/include/gguf.h b/python/freetoken/kernel/csrc/ggml/include/gguf.h
new file mode 100644
index 00000000..b3a1e123
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/include/gguf.h
@@ -0,0 +1,211 @@
+// This file contains functionality related to "GGUF" files, the binary file format used by ggml.
+// GGUF files have the following structure:
+//
+// 1. File magic "GGUF" (4 bytes).
+// 2. File version (uint32_t).
+// 3. Number of ggml tensors in file (int64_t).
+// 4. Number of key-value-pairs in file (int64_t).
+// 5. For each KV pair:
+// 1. The key (string).
+// 2. The value type (gguf_type).
+// 3a. If the value type is GGUF_TYPE_ARRAY:
+// 1. The type of the array (gguf_type).
+// 2. The number of elements in the array (uint64_t).
+// 3. The binary representation of each element in the array.
+// 3b. Otherwise:
+// 1. The binary representation of the value.
+// 6. For each ggml tensor:
+// 1. The tensor name (string).
+// 2. The number of dimensions of the tensor (uint32_t).
+// 3. For each dimension:
+// 1. The size of the tensor in the dimension (int64_t).
+// 4. The tensor data type (ggml_type).
+// 5. The tensor data offset in the tensor data binary blob (uint64_t).
+// 7. The tensor data binary blob (optional, aligned).
+//
+// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator.
+// All enums are stored as int32_t.
+// All bool values are stored as int8_t.
+// If the special key "general.alignment" (uint32_t) is defined it is used for alignment,
+// otherwise GGUF_DEFAULT_ALIGNMENT is used.
+//
+// Module maintainer: Johannes Gäßler (@JohannesGaessler, [email protected])
+
+#pragma once
+
+#include "ggml.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#define GGUF_MAGIC "GGUF"
+#define GGUF_VERSION 3
+
+#define GGUF_KEY_GENERAL_ALIGNMENT "general.alignment"
+
+#define GGUF_DEFAULT_ALIGNMENT 32
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ // types that can be stored as GGUF KV data
+ enum gguf_type {
+ GGUF_TYPE_UINT8 = 0,
+ GGUF_TYPE_INT8 = 1,
+ GGUF_TYPE_UINT16 = 2,
+ GGUF_TYPE_INT16 = 3,
+ GGUF_TYPE_UINT32 = 4,
+ GGUF_TYPE_INT32 = 5,
+ GGUF_TYPE_FLOAT32 = 6,
+ GGUF_TYPE_BOOL = 7,
+ GGUF_TYPE_STRING = 8,
+ GGUF_TYPE_ARRAY = 9,
+ GGUF_TYPE_UINT64 = 10,
+ GGUF_TYPE_INT64 = 11,
+ GGUF_TYPE_FLOAT64 = 12,
+ GGUF_TYPE_COUNT, // marks the end of the enum
+ };
+
+ struct gguf_context;
+
+ struct gguf_init_params {
+ bool no_alloc;
+
+ // if not NULL, create a ggml_context and allocate the tensor data in it
+ struct ggml_context ** ctx;
+ };
+
+ // callback to simulate or wrap a FILE pointer - read up to `len` bytes at `offset` into `output` and return the number of bytes read
+ typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len);
+
+ GGML_API struct gguf_context * gguf_init_empty(void);
+ GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params);
+ GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);
+ GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params);
+
+ // max_chunk_read is the maximum number of bytes that the GGUF code will read at once from the callback, a value of 0 means no limit
+ GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params);
+
+ GGML_API void gguf_free(struct gguf_context * ctx);
+
+ GGML_API const char * gguf_type_name(enum gguf_type type);
+
+ GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx);
+ GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx);
+ GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); // padded to gguf_get_alignment if and only if the gguf_context contains at least one tensor
+
+ GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx);
+ GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found
+ GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id);
+
+ GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id);
+
+ // will abort if the wrong type is used for the key
+ GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id);
+ GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id);
+ GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id);
+ GGML_API size_t gguf_get_arr_n (const struct gguf_context * ctx, int64_t key_id);
+
+ // get raw pointer to the first element of the array with the given key_id
+ // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference)
+ GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id);
+
+ // get ith C string from array with given key_id
+ GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i);
+
+ GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx);
+ GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found
+ GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id);
+ GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id);
+ GGML_API const int64_t * gguf_get_tensor_ne (const struct gguf_context * ctx, int64_t tensor_id); // returns ne, an array of GGML_MAX_DIMS elements; ne[dim] is 1 for dim >= n_dims
+ GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id);
+ GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id);
+
+ // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist)
+ GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key);
+
+ // overrides an existing KV pair or adds a new one, the new KV pair is always at the back
+ GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);
+ GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);
+ GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);
+ GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);
+ GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);
+ GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);
+ GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);
+ GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val);
+ GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val);
+ GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val);
+ GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);
+ GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);
+
+ // creates a new array with n elements of the given type and copies the corresponding number of bytes from data
+ GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n);
+
+ // creates a new array with n strings and copies the corresponding strings from data
+ GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n);
+
+ // set or add KV pairs from another context
+ GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src);
+
+ // add tensor to GGUF context, tensor name must be unique
+ GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);
+
+ // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated
+ // in such a way that the tensor data remains as one contiguous block (except for padding)
+ GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);
+
+ // assumes that at least gguf_get_tensor_size bytes can be read from data
+ GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data);
+
+ // writing gguf files can be done in 3 ways:
+ //
+ // - write the entire gguf_context to a binary file in a single pass:
+ //
+ // gguf_write_to_file(ctx, fname, /*only_meta =*/ false);
+ //
+ // - write only the meta data to a file, then re-open the file and append the tensor data:
+ //
+ // gguf_write_to_file(ctx, fname, /*only_meta =*/ true);
+ // FILE * f = fopen(fname, "ab");
+ // fwrite(f, ...); // write tensor data
+ // fclose(f);
+ //
+ // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
+ //
+ // FILE * f = fopen(fname, "wb");
+ // const size_t size_meta = gguf_get_meta_size(ctx);
+ // fseek(f, size_meta, SEEK_SET);
+ // fwrite(f, ...); // write tensor data
+ // void * data = malloc(size_meta);
+ // gguf_get_meta_data(ctx, data);
+ // rewind(f);
+ // fwrite(data, 1, data, f);
+ // free(data);
+ // fclose(f);
+ //
+
+ // write the entire context to a binary file
+ GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta);
+ GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);
+
+ // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
+ GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);
+
+ // writes the meta data to pointer "data"
+ GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/python/freetoken/kernel/csrc/ggml/src/ggml-common.h b/python/freetoken/kernel/csrc/ggml/src/ggml-common.h
new file mode 100644
index 00000000..1dbbe326
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/src/ggml-common.h
@@ -0,0 +1,1911 @@
+#ifndef GGML_COMMON_DECL
+
+#if defined(GGML_COMMON_DECL_C)
+#include <stdint.h>
+
+typedef uint16_t ggml_half;
+typedef uint32_t ggml_half2;
+
+#define GGML_COMMON_AGGR_U
+#define GGML_COMMON_AGGR_S
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_CPP)
+#include <cstdint>
+
+typedef uint16_t ggml_half;
+typedef uint32_t ggml_half2;
+
+// std-c++ allow anonymous unions but some compiler warn on it
+#define GGML_COMMON_AGGR_U data
+// std-c++ do not allow it.
+#define GGML_COMMON_AGGR_S data
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_METAL)
+#include <metal_stdlib>
+
+typedef half ggml_half;
+typedef half2 ggml_half2;
+
+#define GGML_COMMON_AGGR_U
+#define GGML_COMMON_AGGR_S
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_CUDA)
+#if defined(GGML_COMMON_DECL_MUSA)
+#include <musa_fp16.h>
+#else
+#include <cuda_fp16.h>
+#endif
+#include <cstdint>
+
+typedef half ggml_half;
+typedef half2 ggml_half2;
+
+#define GGML_COMMON_AGGR_U
+#define GGML_COMMON_AGGR_S data
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_HIP)
+#include <hip/hip_fp16.h>
+#include <cstdint>
+
+typedef half ggml_half;
+typedef half2 ggml_half2;
+
+#define GGML_COMMON_AGGR_U
+#define GGML_COMMON_AGGR_S data
+
+#define GGML_COMMON_DECL
+#elif defined(GGML_COMMON_DECL_SYCL)
+#include <sycl/half_type.hpp>
+#include <cstdint>
+
+typedef sycl::half ggml_half;
+typedef sycl::half2 ggml_half2;
+
+#define GGML_COMMON_AGGR_U
+#define GGML_COMMON_AGGR_S data
+
+#define GGML_COMMON_DECL
+#endif
+
+#if defined(GGML_COMMON_DECL)
+
+#ifndef __cplusplus
+#ifndef static_assert
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)
+#define static_assert(cond, msg) _Static_assert(cond, msg)
+#else
+#define static_assert(cond, msg) struct global_scope_noop_trick
+#endif
+#endif
+#endif // __cplusplus
+
+// QK = number of values after dequantization
+// QK_K = super-block size
+
+#define QK_K 256
+#define K_SCALE_SIZE 12
+
+#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL)
+// QR = QK / number of values before dequantization
+// QI = number of 32 bit integers before dequantization
+
+#define QI1_0 (QK1_0 / 32)
+#define QR1_0 1
+
+#define QI2_0 (QK2_0 / 32)
+#define QR2_0 1
+
+
+#define QI4_0 (QK4_0 / (4 * QR4_0))
+#define QR4_0 2
+
+#define QI4_1 (QK4_1 / (4 * QR4_1))
+#define QR4_1 2
+
+#define QI_MXFP4 (QK_MXFP4 / (4 * QR_MXFP4))
+#define QR_MXFP4 2
+
+#define QI_NVFP4 (QK_NVFP4 / (4 * QR_NVFP4))
+#define QR_NVFP4 2
+
+#define QI5_0 (QK5_0 / (4 * QR5_0))
+#define QR5_0 2
+
+#define QI5_1 (QK5_1 / (4 * QR5_1))
+#define QR5_1 2
+
+#define QI8_0 (QK8_0 / (4 * QR8_0))
+#define QR8_0 1
+
+#define QI8_1 (QK8_1 / (4 * QR8_1))
+#define QR8_1 1
+
+#define QI2_K (QK_K / (4*QR2_K))
+#define QR2_K 4
+
+#define QI3_K (QK_K / (4*QR3_K))
+#define QR3_K 4
+
+#define QI4_K (QK_K / (4*QR4_K))
+#define QR4_K 2
+
+#define QI5_K (QK_K / (4*QR5_K))
+#define QR5_K 2
+
+#define QI6_K (QK_K / (4*QR6_K))
+#define QR6_K 2
+
+#define QI2_XXS (QK_K / (4*QR2_XXS))
+#define QR2_XXS 4
+
+#define QI2_XS (QK_K / (4*QR2_XS))
+#define QR2_XS 4
+
+#define QI2_S (QK_K / (4*QR2_S))
+#define QR2_S 4
+
+#define QI3_XXS (QK_K / (4*QR3_XXS))
+#define QR3_XXS 4
+
+#define QI3_XS (QK_K / (4*QR3_XS))
+#define QR3_XS 4
+
+#define QI1_S (QK_K / (4*QR1_S))
+#define QR1_S 8
+
+#define QI1_M (QK_K / (4*QR1_M))
+#define QR1_M 8
+
+#define QI4_NL (QK4_NL / (4*QR4_NL))
+#define QR4_NL 2
+
+#define QI4_XS (QK_K / (4*QR4_XS))
+#define QR4_XS 2
+
+#define QI3_S (QK_K / (4*QR3_S))
+#define QR3_S 4
+
+#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP
+
+#ifdef _MSC_VER
+#define GGML_EXTENSION
+#else // _MSC_VER
+#define GGML_EXTENSION __extension__
+#endif // _MSC_VER
+
+#define QK1_0 128
+typedef struct {
+ ggml_half d; // delta
+ uint8_t qs[QK1_0 / 8]; // bits / quants
+} block_q1_0;
+static_assert(sizeof(block_q1_0) == sizeof(ggml_half) + QK1_0 / 8, "wrong q1_0 block size/padding");
+
+#define QK2_0 64
+typedef struct {
+ ggml_half d; // delta (scale)
+ uint8_t qs[QK2_0 / 4]; // 2 bits per element
+} block_q2_0;
+static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding");
+
+#define QK4_0 32
+typedef struct {
+ ggml_half d; // delta
+ uint8_t qs[QK4_0 / 2]; // nibbles / quants
+} block_q4_0;
+static_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, "wrong q4_0 block size/padding");
+
+#define QK4_1 32
+typedef struct {
+ GGML_EXTENSION union {
+ struct {
+ ggml_half d; // delta
+ ggml_half m; // min
+ } GGML_COMMON_AGGR_S;
+ ggml_half2 dm;
+ } GGML_COMMON_AGGR_U;
+ uint8_t qs[QK4_1 / 2]; // nibbles / quants
+} block_q4_1;
+static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding");
+
+#define QK_MXFP4 32
+typedef struct {
+ uint8_t e; // E8M0
+ uint8_t qs[QK_MXFP4/2];
+} block_mxfp4;
+static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + QK_MXFP4/2, "wrong mxfp4 block size/padding");
+
+#define QK_NVFP4 64
+#define QK_NVFP4_SUB 16 // sub-block size for per-group scales
+typedef struct {
+ uint8_t d[QK_NVFP4/QK_NVFP4_SUB]; // UE4M3 scales (4 bytes, one per 16-element sub-block)
+ uint8_t qs[QK_NVFP4/2]; // packed 4-bit E2M1 values (32 bytes)
+} block_nvfp4;
+static_assert(sizeof(block_nvfp4) == sizeof(uint8_t)*(QK_NVFP4/QK_NVFP4_SUB) + QK_NVFP4/2, "wrong nvfp4 block size/padding");
+
+#define QK5_0 32
+typedef struct {
+ ggml_half d; // delta
+ uint8_t qh[4]; // 5-th bit of quants
+ uint8_t qs[QK5_0 / 2]; // nibbles / quants
+} block_q5_0;
+static_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding");
+
+#define QK5_1 32
+typedef struct {
+ GGML_EXTENSION union {
+ struct {
+ ggml_half d; // delta
+ ggml_half m; // min
+ } GGML_COMMON_AGGR_S;
+ ggml_half2 dm;
+ } GGML_COMMON_AGGR_U;
+ uint8_t qh[4]; // 5-th bit of quants
+ uint8_t qs[QK5_1 / 2]; // nibbles / quants
+} block_q5_1;
+static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding");
+
+#define QK8_0 32
+typedef struct {
+ ggml_half d; // delta
+ int8_t qs[QK8_0]; // quants
+} block_q8_0;
+static_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, "wrong q8_0 block size/padding");
+
+#define QK8_1 32
+typedef struct {
+ GGML_EXTENSION union {
+ struct {
+ ggml_half d; // delta
+ ggml_half s; // d * sum(qs[i])
+ } GGML_COMMON_AGGR_S;
+ ggml_half2 ds;
+ } GGML_COMMON_AGGR_U;
+ int8_t qs[QK8_1]; // quants
+} block_q8_1;
+static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding");
+
+//
+// Ternary quantization
+//
+
+// 1.6875 bpw
+typedef struct {
+ uint8_t qs[(QK_K - 4 * QK_K / 64) / 5]; // 5 elements per byte (3^5 = 243 < 256)
+ uint8_t qh[QK_K/64]; // 4 elements per byte
+ ggml_half d;
+} block_tq1_0;
+static_assert(sizeof(block_tq1_0) == sizeof(ggml_half) + QK_K / 64 + (QK_K - 4 * QK_K / 64) / 5, "wrong tq1_0 block size/padding");
+
+// 2.0625 bpw
+typedef struct {
+ uint8_t qs[QK_K/4]; // 2 bits per element
+ ggml_half d;
+} block_tq2_0;
+static_assert(sizeof(block_tq2_0) == sizeof(ggml_half) + QK_K / 4, "wrong tq2_0 block size/padding");
+
+//
+// Super-block quantization structures
+//
+
+// 2-bit quantization
+// weight is represented as x = a * q + b
+// 16 blocks of 16 elements each
+// Effectively 2.625 bits per weight
+typedef struct {
+ uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
+ uint8_t qs[QK_K/4]; // quants
+ GGML_EXTENSION union {
+ struct {
+ ggml_half d; // super-block scale for quantized scales
+ ggml_half dmin; // super-block scale for quantized mins
+ } GGML_COMMON_AGGR_S;
+ ggml_half2 dm;
+ } GGML_COMMON_AGGR_U;
+} block_q2_K;
+static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");
+
+// 3-bit quantization
+// weight is represented as x = a * q
+// 16 blocks of 16 elements each
+// Effectively 3.4375 bits per weight
+typedef struct {
+ uint8_t hmask[QK_K/8]; // quants - high bit
+ uint8_t qs[QK_K/4]; // quants - low 2 bits
+ uint8_t scales[12]; // scales, quantized with 6 bits
+ ggml_half d; // super-block scale
+} block_q3_K;
+static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");
+
+// 4-bit quantization
+// 8 blocks of 32 elements each
+// weight is represented as x = a * q + b
+// Effectively 4.5 bits per weight
+typedef struct {
+ GGML_EXTENSION union {
+ struct {
+ ggml_half d; // super-block scale for quantized scales
+ ggml_half dmin; // super-block scale for quantized mins
+ } GGML_COMMON_AGGR_S;
+ ggml_half2 dm;
+ } GGML_COMMON_AGGR_U;
+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
+ uint8_t qs[QK_K/2]; // 4--bit quants
+} block_q4_K;
+static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding");
+
+// 5-bit quantization
+// 8 blocks of 32 elements each
+// weight is represented as x = a * q + b
+// Effectively 5.5 bits per weight
+typedef struct {
+ GGML_EXTENSION union {
+ struct {
+ ggml_half d; // super-block scale for quantized scales
+ ggml_half dmin; // super-block scale for quantized mins
+ } GGML_COMMON_AGGR_S;
+ ggml_half2 dm;
+ } GGML_COMMON_AGGR_U;
+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
+ uint8_t qh[QK_K/8]; // quants, high bit
+ uint8_t qs[QK_K/2]; // quants, low 4 bits
+} block_q5_K;
+static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");
+
+// 6-bit quantization
+// weight is represented as x = a * q
+// 16 blocks of 16 elements each
+// Effectively 6.5625 bits per weight
+typedef struct {
+ uint8_t ql[QK_K/2]; // quants, lower 4 bits
+ uint8_t qh[QK_K/4]; // quants, upper 2 bits
+ int8_t scales[QK_K/16]; // scales, quantized with 8 bits
+ ggml_half d; // super-block scale
+} block_q6_K;
+static_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");
+
+// This is only used for intermediate quantization and dot products
+typedef struct {
+ float d; // delta
+ int8_t qs[QK_K]; // quants
+ int16_t bsums[QK_K/16]; // sum of quants in groups of 16
+} block_q8_K;
+static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");
+
+// (Almost) "true" 2-bit quantization.
+// Due to the need to use blocks as per ggml design, it ends up using
+// 2.0625 bpw because of the 16-bit scale for each block of 256.
+typedef struct {
+ ggml_half d;
+ uint16_t qs[QK_K/8];
+} block_iq2_xxs;
+static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");
+
+// 2.3125 bpw quants
+typedef struct {
+ ggml_half d;
+ uint16_t qs[QK_K/8];
+ uint8_t scales[QK_K/32];
+} block_iq2_xs;
+static_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");
+
+// 2.5625 bpw quants
+typedef struct {
+ ggml_half d;
+ uint8_t qs[QK_K/4];
+ uint8_t qh[QK_K/32];
+ uint8_t scales[QK_K/32];
+} block_iq2_s;
+static_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding");
+
+// (Almost) "true" 3-bit quantization.
+// Due to the need to use blocks as per ggml design, it ends up using
+// 3.0625 bpw because of the 16-bit scale for each block of 256.
+typedef struct {
+ ggml_half d;
+ uint8_t qs[3*QK_K/8];
+} block_iq3_xxs;
+static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");
+
+// 3.4375 bpw
+#define IQ3S_N_SCALE QK_K/64
+typedef struct {
+ ggml_half d;
+ uint8_t qs[QK_K/4];
+ uint8_t qh[QK_K/32];
+ uint8_t signs[QK_K/8];
+ uint8_t scales[IQ3S_N_SCALE];
+} block_iq3_s;
+static_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding");
+
+// 1.5625 bpw
+typedef struct {
+ ggml_half d;
+ uint8_t qs[QK_K/8];
+ uint16_t qh[QK_K/32];
+} block_iq1_s;
+static_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");
+
+// 1.75 bpw
+typedef struct {
+ uint8_t qs[QK_K/8]; // grid index, low 8 bits
+ uint8_t qh[QK_K/16]; // grid index, high 3 bits + grid shift bit (for two groups of 8)
+ uint8_t scales[QK_K/32]; // 3-bit block scales (4-bit if QK_K == 64)
+} block_iq1_m;
+static_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, "wrong iq1_m block size/padding");
+
+// Used by IQ1_M quants
+typedef union {
+ ggml_half f16;
+ uint16_t u16;
+} iq1m_scale_t;
+
+// Non-linear quants
+#define QK4_NL 32
+typedef struct {
+ ggml_half d;
+ uint8_t qs[QK4_NL/2];
+} block_iq4_nl;
+static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "wrong iq4_nl block size/padding");
+
+typedef struct {
+ ggml_half d;
+ uint16_t scales_h;
+ uint8_t scales_l[QK_K/64];
+ uint8_t qs[QK_K/2];
+} block_iq4_xs;
+static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");
+
+#endif // GGML_COMMON_DECL
+#endif // GGML_COMMON_DECL
+
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef GGML_COMMON_IMPL
+
+#if defined(GGML_COMMON_IMPL_C)
+#include <stdint.h>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#elif defined(GGML_COMMON_IMPL_CPP)
+#include <cstdint>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#elif defined(GGML_COMMON_IMPL_METAL)
+#include <metal_stdlib>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP) || defined(GGML_COMMON_IMPL_MUSA)
+#include <cstdint>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#elif defined(GGML_COMMON_IMPL_SYCL)
+
+#include <cstdint>
+
+#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
+#define GGML_TABLE_END() };
+
+#define GGML_COMMON_IMPL
+#endif
+
+#if defined(GGML_COMMON_IMPL)
+
+GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)
+ 1, 2, 4, 8, 16, 32, 64, 128
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)
+ 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,
+ 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,
+ 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,
+ 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,
+ 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,
+ 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,
+ 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,
+ 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)
+ 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,
+ 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,
+ 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,
+ 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,
+ 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,
+ 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,
+ 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,
+ 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,
+ 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,
+ 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,
+ 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,
+ 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,
+ 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,
+ 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,
+ 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,
+ 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,
+ 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,
+ 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,
+ 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,
+ 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,
+ 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,
+ 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,
+ 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,
+ 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,
+ 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,
+ 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,
+ 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,
+ 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,
+ 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,
+ 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,
+ 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,
+ 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,
+GGML_TABLE_END()
+
+
+GGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256)
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,
+ 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,
+ 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,
+ 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,
+ 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,
+ 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,
+ 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,
+ 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,
+ 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,
+ 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,
+ 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,
+ 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,
+ 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,
+ 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,
+ 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,
+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,
+ 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,
+ 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,
+ 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,
+ 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,
+ 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,
+ 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,
+ 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,
+ 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,
+ 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,
+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,
+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,
+ 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,
+ 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,
+ 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,
+ 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,
+ 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,
+ 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,
+ 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,
+ 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,
+ 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,
+ 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,
+ 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,
+ 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,
+ 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,
+ 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,
+ 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,
+ 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,
+ 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,
+ 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,
+ 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,
+ 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,
+ 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,
+ 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,
+ 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,
+ 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,
+ 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,
+ 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,
+ 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,
+ 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,
+ 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,
+ 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,
+ 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,
+ 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,
+ 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,
+ 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,
+ 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,
+ 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512)
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,
+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,
+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,
+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,
+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,
+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,
+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,
+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,
+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,
+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,
+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,
+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,
+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,
+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,
+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,
+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,
+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,
+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,
+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,
+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,
+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,
+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,
+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,
+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,
+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,
+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,
+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,
+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,
+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,
+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,
+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,
+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,
+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,
+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,
+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,
+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,
+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,
+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,
+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,
+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,
+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,
+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,
+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,
+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,
+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,
+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,
+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,
+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,
+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,
+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,
+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,
+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,
+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,
+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,
+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,
+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,
+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,
+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,
+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,
+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,
+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,
+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,
+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,
+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,
+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,
+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,
+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,
+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,
+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,
+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,
+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,
+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,
+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,
+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,
+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,
+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,
+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,
+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,
+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,
+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,
+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,
+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,
+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,
+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,
+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,
+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,
+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,
+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,
+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,
+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,
+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,
+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,
+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,
+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,
+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,
+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,
+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,
+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,
+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,
+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,
+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,
+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,
+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,
+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,
+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,
+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,
+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,
+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,
+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,
+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,
+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,
+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,
+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,
+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,
+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,
+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,
+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,
+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,
+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,
+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,
+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,
+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,
+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,
+ 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,
+ 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,
+ 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,
+ 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,
+ 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,
+ 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,
+ 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,
+ 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,
+ 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,
+ 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,
+ 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,
+ 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,
+ 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,
+ 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,
+ 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,
+ 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,
+ 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,
+ 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,
+ 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,
+ 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,
+ 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,
+ 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,
+ 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,
+ 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,
+ 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,
+ 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,
+ 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,
+ 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,
+ 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,
+ 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,
+ 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,
+ 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,
+ 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,
+ 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,
+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,
+ 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,
+ 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,
+ 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,
+ 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,
+ 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,
+ 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,
+ 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,
+ 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,
+ 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,
+ 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,
+ 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,
+ 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,
+ 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,
+ 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,
+ 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,
+ 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,
+ 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,
+ 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,
+ 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,
+ 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,
+ 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,
+ 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,
+ 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,
+ 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,
+ 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,
+ 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,
+ 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,
+ 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,
+ 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,
+ 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,
+ 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,
+ 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,
+ 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,
+ 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,
+ 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,
+ 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,
+ 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,
+ 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,
+ 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,
+ 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,
+ 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,
+ 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,
+ 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,
+ 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,
+ 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,
+ 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,
+ 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,
+ 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,
+ 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,
+ 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,
+ 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,
+ 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,
+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,
+ 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,
+ 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,
+ 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,
+ 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,
+ 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,
+ 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,
+ 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,
+ 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,
+ 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,
+ 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,
+ 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,
+ 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,
+ 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,
+ 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,
+ 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,
+ 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,
+ 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,
+ 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,
+ 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,
+ 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,
+ 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,
+ 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,
+ 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,
+ 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,
+ 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,
+ 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,
+ 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,
+ 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,
+ 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,
+ 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,
+ 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,
+ 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,
+ 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,
+ 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,
+ 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,
+ 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,
+ 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,
+ 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,
+ 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,
+ 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,
+ 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,
+ 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,
+ 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,
+ 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,
+ 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,
+ 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,
+ 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,
+ 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,
+ 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,
+ 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,
+ 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,
+ 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,
+ 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,
+ 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,
+ 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,
+ 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,
+ 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,
+ 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,
+ 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,
+ 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,
+ 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,
+ 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,
+ 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,
+ 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,
+ 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,
+ 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,
+ 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,
+ 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,
+ 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,
+ 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,
+ 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,
+ 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,
+ 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,
+ 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,
+ 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,
+ 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,
+ 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,
+ 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,
+ 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,
+ 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,
+ 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,
+ 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,
+ 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,
+ 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,
+ 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,
+ 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,
+ 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,
+ 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,
+ 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,
+ 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,
+ 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,
+ 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,
+ 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,
+ 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,
+ 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,
+ 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,
+ 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,
+ 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,
+ 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,
+ 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,
+ 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,
+ 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,
+ 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,
+ 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,
+ 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,
+ 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,
+ 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,
+ 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,
+ 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,
+ 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,
+ 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,
+ 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,
+ 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,
+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,
+ 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,
+ 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,
+ 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,
+ 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,
+ 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,
+ 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,
+ 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,
+ 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,
+ 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,
+ 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,
+ 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,
+ 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,
+ 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,
+ 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,
+ 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,
+ 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,
+ 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,
+ 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,
+ 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,
+ 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,
+ 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,
+ 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,
+ 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,
+ 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,
+ 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,
+ 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,
+ 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,
+ 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,
+ 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,
+ 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,
+ 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,
+ 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,
+ 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,
+ 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,
+ 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,
+ 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,
+ 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,
+ 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,
+ 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,
+ 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,
+ 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,
+ 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,
+ 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,
+ 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,
+ 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,
+ 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,
+ 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,
+ 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256)
+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,
+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,
+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,
+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,
+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,
+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,
+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,
+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,
+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,
+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,
+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,
+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,
+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,
+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,
+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,
+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,
+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,
+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,
+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,
+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,
+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,
+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,
+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,
+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,
+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,
+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,
+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,
+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,
+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,
+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,
+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,
+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,
+GGML_TABLE_END()
+
+GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512)
+ 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,
+ 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,
+ 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,
+ 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,
+ 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,
+ 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,
+ 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,
+ 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,
+ 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,
+ 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,
+ 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,
+ 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,
+ 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,
+ 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,
+ 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,
+ 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,
+ 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,
+ 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,
+ 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,
+ 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,
+ 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,
+ 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,
+ 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,
+ 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,
+ 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,
+ 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,
+ 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,
+ 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,
+ 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,
+ 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,
+ 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,
+ 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,
+ 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,
+ 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,
+ 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,
+ 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,
+ 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,
+ 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,
+ 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,
+ 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,
+ 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,
+ 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,
+ 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,
+ 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,
+ 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,
+ 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,
+ 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,
+ 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,
+ 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,
+ 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,
+ 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,
+ 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,
+ 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,
+ 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,
+ 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,
+ 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,
+ 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,
+ 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,
+ 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,
+ 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,
+ 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,
+ 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,
+ 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,
+ 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,
+GGML_TABLE_END()
+
+// TODO: fix name to kvalues_iq4_nl
+GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16)
+ -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,
+GGML_TABLE_END()
+
+// e2m1 values (doubled), shared by MXFP4 and NVFP4
+// ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
+GGML_TABLE_BEGIN(int8_t, kvalues_fp4, 16)
+ 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12,
+GGML_TABLE_END()
+#define kvalues_mxfp4 kvalues_fp4
+
+#define NGRID_IQ1S 2048
+#define IQ1S_DELTA 0.125f
+#define IQ1M_DELTA 0.125f
+#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP)
+GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)
+ 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,
+ 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,
+ 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000,
+ 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000,
+ 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101,
+ 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff,
+ 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00,
+ 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101,
+ 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100,
+ 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00,
+ 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000,
+ 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000,
+ 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101,
+ 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff,
+ 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100,
+ 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01,
+ 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000,
+ 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff,
+ 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001,
+ 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000,
+ 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00,
+ 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff,
+ 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff,
+ 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000,
+ 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff,
+ 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff,
+ 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101,
+ 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff,
+ 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000,
+ 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100,
+ 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01,
+ 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00,
+ 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00,
+ 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01,
+ 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff,
+ 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000,
+ 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff,
+ 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000,
+ 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101,
+ 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100,
+ 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff,
+ 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff,
+ 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001,
+ 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01,
+ 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff,
+ 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000,
+ 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000,
+ 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff,
+ 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01,
+ 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00,
+ 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000,
+ 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000,
+ 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000,
+ 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001,
+ 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff,
+ 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01,
+ 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff,
+ 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff,
+ 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000,
+ 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000,
+ 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00,
+ 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001,
+ 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100,
+ 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101,
+ 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00,
+ 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00,
+ 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000,
+ 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001,
+ 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff,
+ 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000,
+ 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000,
+ 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff,
+ 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100,
+ 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000,
+ 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101,
+ 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001,
+ 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000,
+ 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff,
+ 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01,
+ 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100,
+ 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000,
+ 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01,
+ 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101,
+ 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000,
+ 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff,
+ 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff,
+ 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001,
+ 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001,
+ 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000,
+ 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000,
+ 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00,
+ 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100,
+ 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01,
+ 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00,
+ 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff,
+ 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000,
+ 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00,
+ 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000,
+ 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff,
+ 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00,
+ 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000,
+ 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000,
+ 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff,
+ 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00,
+ 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00,
+ 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001,
+ 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001,
+ 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000,
+ 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100,
+ 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101,
+ 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101,
+ 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000,
+ 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00,
+ 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff,
+ 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000,
+ 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff,
+ 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff,
+ 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff,
+ 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101,
+ 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001,
+ 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100,
+ 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101,
+ 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01,
+ 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00,
+ 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00,
+ 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000,
+ 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101,
+ 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100,
+ 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001,
+ 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff,
+ 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00,
+ 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000,
+ 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100,
+ 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00,
+ 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01,
+ 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff,
+ 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101,
+ 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100,
+ 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100,
+ 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000,
+ 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff,
+ 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff,
+ 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000,
+ 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101,
+ 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000,
+ 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101,
+ 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101,
+ 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100,
+ 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01,
+ 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001,
+ 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001,
+ 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff,
+ 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff,
+ 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000,
+ 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000,
+ 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101,
+ 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff,
+ 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001,
+ 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000,
+ 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff,
+ 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00,
+ 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff,
+ 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000,
+ 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00,
+ 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff,
+ 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000,
+ 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000,
+ 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff,
+ 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000,
+ 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000,
+ 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000,
+ 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101,
+ 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff,
+ 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100,
+ 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101,
+ 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001,
+ 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff,
+ 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100,
+ 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff,
+ 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01,
+ 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff,
+ 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff,
+ 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff,
+ 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff,
+ 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001,
+ 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff,
+ 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01,
+ 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00,
+ 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100,
+ 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101,
+ 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff,
+ 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00,
+ 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01,
+ 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00,
+ 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100,
+ 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00,
+ 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000,
+ 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000,
+ 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000,
+ 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000,
+ 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001,
+ 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff,
+ 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00,
+ 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff,
+ 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00,
+ 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000,
+ 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff,
+ 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff,
+ 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101,
+ 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000,
+ 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff,
+ 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff,
+ 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff,
+ 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff,
+ 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000,
+ 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000,
+ 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100,
+ 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00,
+ 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100,
+ 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100,
+ 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00,
+ 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff,
+ 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff,
+ 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100,
+ 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff,
+ 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000,
+ 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00,
+ 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,
+ 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00,
+ 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100,
+ 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff,
+ 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff,
+ 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001,
+ 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00,
+ 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101,
+ 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001,
+ 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000,
+ 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100,
+ 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000,
+ 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001,
+ 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000,
+ 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff,
+ 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101,
+ 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,
+ 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101,
+ 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000,
+ 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff,
+ 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000,
+ 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff,
+ 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01,
+ 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100,
+ 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff,
+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101,
+ 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001,
+ 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01,
+ 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff,
+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,
+ 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff,
+ 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00,
+ 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff,
+ 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff,
+ 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff,
+ 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001,
+ 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff,
+ 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff,
+ 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001,
+ 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff,
+ 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff,
+ 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000,
+ 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00,
+ 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100,
+ 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01,
+ 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff,
+ 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff,
+ 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000,
+ 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101,
+ 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01,
+ 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100,
+ 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100,
+ 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00,
+ 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101,
+ 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001,
+ 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01,
+ 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100,
+ 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff,
+ 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01,
+ 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff,
+ 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000,
+ 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001,
+ 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01,
+ 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff,
+ 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff,
+ 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00,
+ 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff,
+ 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100,
+ 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000,
+ 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001,
+ 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100,
+ 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff,
+ 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff,
+ 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01,
+ 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101,
+ 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001,
+ 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff,
+ 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101,
+ 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100,
+ 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00,
+ 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01,
+ 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001,
+ 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00,
+ 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000,
+ 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101,
+ 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001,
+ 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100,
+ 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000,
+ 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000,
+ 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00,
+ 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101,
+ 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff,
+ 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101,
+ 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001,
+ 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01,
+ 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100,
+ 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000,
+ 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00,
+ 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff,
+ 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001,
+ 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001,
+ 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff,
+ 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00,
+ 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100,
+ 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00,
+ 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101,
+ 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff,
+ 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff,
+ 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff,
+ 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00,
+ 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff,
+ 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff,
+ 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101,
+ 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100,
+ 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101,
+ 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00,
+ 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00,
+ 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff,
+ 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff,
+ 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001,
+ 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000,
+ 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff,
+ 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff,
+ 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff,
+ 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff,
+ 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001,
+ 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100,
+ 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101,
+ 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001,
+ 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001,
+ 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101,
+ 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101,
+ 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff,
+ 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff,
+ 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000,
+ 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101,
+ 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001,
+ 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff,
+ 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000,
+ 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff,
+ 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100,
+ 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000,
+ 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101,
+ 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff,
+ 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100,
+ 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff,
+ 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01,
+ 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100,
+ 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000,
+ 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100,
+ 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100,
+ 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001,
+ 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000,
+ 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000,
+ 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000,
+ 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00,
+ 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff,
+ 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001,
+ 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000,
+ 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101,
+ 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101,
+ 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000,
+ 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff,
+ 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000,
+ 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101,
+ 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff,
+ 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff,
+ 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000,
+ 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101,
+ 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff,
+ 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff,
+ 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01,
+ 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff,
+ 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000,
+ 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101,
+ 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff,
+ 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff,
+ 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff,
+ 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01,
+ 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00,
+ 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000,
+ 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000,
+ 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101,
+ 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001,
+ 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff,
+ 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000,
+ 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff,
+ 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000,
+ 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000,
+ 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000,
+ 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000,
+ 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001,
+ 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff,
+ 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001,
+ 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000,
+ 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000,
+ 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00,
+ 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000,
+ 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101,
+ 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001,
+ 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00,
+ 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001,
+ 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff,
+ 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000,
+ 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00,
+ 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100,
+ 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101,
+ 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001,
+ 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01,
+ 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff,
+ 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff,
+ 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00,
+ 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01,
+ 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100,
+ 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000,
+ 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff,
+ 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff,
+ 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff,
+ 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000,
+ 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff,
+ 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101,
+ 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff,
+ 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001,
+ 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001,
+ 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001,
+ 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000,
+ 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000,
+ 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff,
+ 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101,
+ 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001,
+ 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff,
+ 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000,
+ 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000,
+ 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff,
+ 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101,
+ 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000,
+ 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100,
+ 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00,
+ 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000,
+ 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff,
+ 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01,
+ 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00,
+ 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff,
+ 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000,
+ 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101,
+ 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff,
+ 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001,
+ 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff,
+ 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000,
+ 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100,
+ 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff,
+ 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101,
+ 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100,
+ 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff,
+ 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01,
+ 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000,
+ 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff,
+ 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00,
+ 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100,
+ 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff,
+ 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001,
+ 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00,
+ 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100,
+ 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff,
+ 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01,
+ 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00,
+ 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000,
+ 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01,
+ 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff,
+ 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000,
+ 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff,
+ 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff,
+ 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff,
+ 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001,
+ 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff,
+ 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01,
+ 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff,
+ 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00,
+ 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00,
+ 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001,
+ 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101,
+ 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101,
+ 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff,
+ 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000,
+ 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101,
+GGML_TABLE_END()
+#else
+GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S)
+ 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,
+ 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,
+ 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,
+ 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,
+ 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,
+ 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,
+ 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,
+ 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,
+ 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,
+ 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,
+ 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,
+ 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,
+ 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,
+ 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,
+ 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,
+ 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,
+ 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,
+ 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,
+ 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,
+ 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,
+ 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,
+ 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,
+ 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,
+ 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,
+ 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,
+ 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,
+ 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,
+ 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,
+ 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,
+ 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,
+ 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,
+ 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,
+ 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,
+ 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,
+ 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,
+ 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,
+ 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,
+ 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,
+ 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,
+ 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,
+ 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,
+ 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,
+ 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,
+ 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,
+ 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,
+ 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,
+ 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,
+ 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,
+ 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,
+ 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,
+ 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,
+ 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,
+ 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,
+ 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,
+ 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,
+ 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,
+ 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,
+ 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,
+ 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,
+ 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,
+ 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,
+ 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,
+ 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,
+ 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,
+ 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,
+ 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,
+ 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,
+ 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,
+ 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,
+ 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,
+ 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,
+ 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,
+ 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,
+ 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,
+ 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,
+ 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,
+ 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,
+ 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,
+ 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,
+ 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,
+ 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,
+ 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,
+ 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,
+ 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,
+ 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,
+ 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,
+ 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,
+ 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,
+ 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,
+ 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,
+ 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,
+ 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,
+ 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,
+ 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,
+ 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,
+ 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,
+ 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,
+ 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,
+ 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,
+ 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,
+ 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,
+ 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,
+ 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,
+ 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,
+ 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,
+ 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,
+ 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,
+ 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,
+ 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,
+ 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,
+ 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,
+ 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,
+ 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,
+ 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,
+ 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,
+ 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,
+ 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,
+ 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,
+ 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,
+ 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,
+ 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,
+ 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,
+ 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,
+ 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,
+ 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,
+ 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,
+ 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,
+ 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,
+ 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,
+ 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,
+ 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,
+ 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,
+ 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,
+ 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,
+ 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,
+ 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,
+ 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,
+ 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,
+ 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,
+ 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,
+ 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,
+ 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,
+ 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,
+ 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,
+ 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,
+ 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,
+ 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,
+ 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,
+ 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,
+ 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,
+ 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,
+ 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,
+ 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,
+ 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,
+ 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,
+ 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,
+ 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,
+ 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,
+ 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,
+ 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,
+ 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,
+ 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,
+ 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,
+ 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,
+ 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,
+ 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,
+ 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,
+ 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,
+ 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,
+ 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,
+ 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,
+ 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,
+ 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,
+ 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,
+ 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,
+ 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,
+ 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,
+ 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,
+ 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,
+ 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,
+ 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,
+ 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,
+ 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,
+ 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,
+ 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,
+ 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,
+ 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,
+ 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,
+ 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,
+ 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,
+ 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,
+ 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,
+ 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,
+ 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,
+ 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,
+ 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,
+ 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,
+ 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,
+ 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,
+ 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,
+ 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,
+ 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,
+ 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,
+ 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,
+ 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,
+ 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,
+ 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,
+ 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,
+ 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,
+ 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,
+ 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,
+ 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,
+ 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,
+ 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,
+ 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,
+ 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,
+ 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,
+ 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,
+ 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,
+ 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,
+ 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,
+ 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,
+ 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,
+ 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,
+ 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,
+ 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,
+ 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,
+ 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,
+ 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,
+ 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,
+ 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,
+ 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,
+ 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,
+ 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,
+ 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,
+ 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,
+ 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,
+ 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,
+ 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,
+ 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,
+ 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,
+ 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,
+ 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,
+ 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,
+ 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,
+ 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,
+ 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,
+ 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,
+ 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,
+ 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,
+ 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,
+ 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,
+ 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,
+ 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,
+ 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,
+ 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,
+GGML_TABLE_END()
+#endif
+
+#endif // GGML_COMMON_IMPL
+#endif // GGML_COMMON_IMPL
diff --git a/python/freetoken/kernel/csrc/ggml/src/ggml-impl.h b/python/freetoken/kernel/csrc/ggml/src/ggml-impl.h
new file mode 100644
index 00000000..ae26e0c2
--- /dev/null
+++ b/python/freetoken/kernel/csrc/ggml/src/ggml-impl.h
@@ -0,0 +1,795 @@
+#pragma once
+
+// GGML internal header
+
+#include "ggml.h"
+#include "gguf.h"
+
+#include <assert.h>
+#include <math.h>
+#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+
+#ifdef __ARM_FEATURE_SVE
+#include <arm_sve.h>
+#endif // __ARM_FEATURE_SVE
+
+#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__)
+// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:
+//
+// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/
+//
+#include <arm_neon.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void ggml_print_backtrace(void);
+
+uint64_t ggml_graph_next_uid(void);
+
+#ifndef MIN
+# define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef MAX
+# define MAX(a, b) ((a) > (b) ? (a) : (b))
+#endif
+
+// required for mmap as gguf only guarantees 32-byte alignment
+#define TENSOR_ALIGNMENT 32
+
+// static_assert should be a #define, but if it's not,
+// fall back to the _Static_assert C11 keyword.
+// if C99 - static_assert is noop
+// ref: https://stackoverflow.com/a/53923785/4039976
+#ifndef __cplusplus
+ #ifndef static_assert
+ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)
+ #define static_assert(cond, msg) _Static_assert(cond, msg)
+ #else
+ #define static_assert(cond, msg) struct global_scope_noop_trick
+ #endif
+ #endif
+#endif
+
+static inline int ggml_up32(int n) {
+ return (n + 31) & ~31;
+}
+
+//static inline int ggml_up64(int n) {
+// return (n + 63) & ~63;
+//}
+
+static inline int ggml_up(int n, int m) {
+ // assert m is a power of 2
+ GGML_ASSERT((m & (m - 1)) == 0);
+ return (n + m - 1) & ~(m - 1);
+}
+
+// TODO: move to ggml.h? (won't be able to inline)
+static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {
+ if (a->type != b->type) {
+ return false;
+ }
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
+ if (a->ne[i] != b->ne[i]) {
+ return false;
+ }
+ if (a->nb[i] != b->nb[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool ggml_op_is_empty(enum ggml_op op) {
+ switch (op) {
+ case GGML_OP_NONE:
+ case GGML_OP_RESHAPE:
+ case GGML_OP_TRANSPOSE:
+ case GGML_OP_VIEW:
+ case GGML_OP_PERMUTE:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static inline bool ggml_impl_is_view(const struct ggml_tensor * t) {
+ return t->view_src != NULL;
+}
+
+static inline float ggml_compute_softplus_f32(float input) {
+ return (input > 20.0f) ? input : logf(1 + expf(input));
+}
+//
+// logging
+//
+
+GGML_ATTRIBUTE_FORMAT(2, 3)
+GGML_API void ggml_log_internal (enum ggml_log_level level, const char * format, ...);
+GGML_API void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data);
+
+#define GGML_LOG(...) ggml_log_internal(GGML_LOG_LEVEL_NONE , __VA_ARGS__)
+#define GGML_LOG_INFO(...) ggml_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)
+#define GGML_LOG_WARN(...) ggml_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)
+#define GGML_LOG_ERROR(...) ggml_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
+#define GGML_LOG_DEBUG(...) ggml_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
+#define GGML_LOG_CONT(...) ggml_log_internal(GGML_LOG_LEVEL_CONT , __VA_ARGS__)
+
+#define GGML_DEBUG 0
+
+#if (GGML_DEBUG >= 1)
+#define GGML_PRINT_DEBUG(...) GGML_LOG_DEBUG(__VA_ARGS__)
+#else
+#define GGML_PRINT_DEBUG(...)
+#endif
+
+#if (GGML_DEBUG >= 5)
+#define GGML_PRINT_DEBUG_5(...) GGML_LOG_DEBUG(__VA_ARGS__)
+#else
+#define GGML_PRINT_DEBUG_5(...)
+#endif
+
+#if (GGML_DEBUG >= 10)
+#define GGML_PRINT_DEBUG_10(...) GGML_LOG_DEBUG(__VA_ARGS__)
+#else
+#define GGML_PRINT_DEBUG_10(...)
+#endif
+
+// tensor params
+
+static void ggml_set_op_params(struct ggml_tensor * tensor, const void * params, size_t params_size) {
+ GGML_ASSERT(tensor != NULL); // silence -Warray-bounds warnings
+ assert(params_size <= GGML_MAX_OP_PARAMS);
+ memcpy(tensor->op_params, params, params_size);
+}
+
+static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {
+ assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));
+ return ((const int32_t *)(tensor->op_params))[i];
+}
+
+static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t i) {
+ assert(i < GGML_MAX_OP_PARAMS / sizeof(float));
+ return ((const float *)(tensor->op_params))[i];
+}
+
+// [TAG_GGML_PREC]
+// - GGML_OP_MUL_MAT
+// 0 - acc
+// 1 - hint
+// 2 - src0 precision
+// 3 - src1 precision
+//
+// - GGML_OP_MUL_MAT_ID
+// 0 - acc
+// 1 - hint
+// 2 - src0 precision
+// 3 - src1 precision
+static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {
+ assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));
+ ((int32_t *)(tensor->op_params))[i] = value;
+}
+
+static void ggml_set_op_params_f32(struct ggml_tensor * tensor, uint32_t i, float value) {
+ assert(i < GGML_MAX_OP_PARAMS / sizeof(float));
+ ((float *)(tensor->op_params))[i] = value;
+}
+
+struct ggml_map_custom1_op_params {
+ ggml_custom1_op_t fun;
+ int n_tasks;
+ void * userdata;
+};
+
+struct ggml_map_custom2_op_params {
+ ggml_custom2_op_t fun;
+ int n_tasks;
+ void * userdata;
+};
+
+struct ggml_map_custom3_op_params {
+ ggml_custom3_op_t fun;
+ int n_tasks;
+ void * userdata;
+};
+
+struct ggml_custom_op_params {
+ ggml_custom_op_t fun;
+ int n_tasks;
+ void * userdata;
+};
+
+// bitset
+
+typedef uint32_t ggml_bitset_t;
+
+static_assert(sizeof(ggml_bitset_t) == 4, "bitset_t constants must be updated");
+#define BITSET_SHR 5 // log2(sizeof(ggml_bitset_t)*8)
+#define BITSET_MASK (sizeof(ggml_bitset_t)*8 - 1)
+
+static size_t ggml_bitset_size(size_t n) {
+ return (n + BITSET_MASK) >> BITSET_SHR;
+}
+
+static inline bool ggml_bitset_get(const ggml_bitset_t * bitset, size_t i) {
+ return !!(bitset[i >> BITSET_SHR] & (1u << (i & BITSET_MASK)));
+}
+
+static inline void ggml_bitset_set(ggml_bitset_t * bitset, size_t i) {
+ bitset[i >> BITSET_SHR] |= (1u << (i & BITSET_MASK));
+}
+
+static inline void ggml_bitset_clear(ggml_bitset_t * bitset, size_t i) {
+ bitset[i >> BITSET_SHR] &= ~(1u << (i & BITSET_MASK));
+}
+
+// hash set
+
+#define GGML_HASHSET_FULL ((size_t)-1)
+#define GGML_HASHSET_ALREADY_EXISTS ((size_t)-2)
+
+struct ggml_hash_set {
+ size_t size;
+ ggml_bitset_t * used; // whether or not the keys are in use i.e. set
+ struct ggml_tensor ** keys; // actual tensors in the set, keys[i] is only defined if ggml_bitset_get(used, i)
+};
+
+struct ggml_hash_set ggml_hash_set_new(size_t size);
+void ggml_hash_set_free(struct ggml_hash_set * hash_set);
+
+// returns the minimum size for a hash set that can hold min_sz elements
+size_t ggml_hash_size(size_t min_sz);
+
+// remove all elements from the hash set
+void ggml_hash_set_reset(struct ggml_hash_set * hash_set);
+
+// returns true if key is in the hash set
+static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key);
+
+// returns GGML_HASHSET_FULL if table is full, otherwise the current index of the key or where it should be inserted
+static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key);
+
+// returns GGML_HASHSET_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full
+static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);
+
+// return index, asserts if table is full
+static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);
+
+// hash function for ggml_tensor
+static inline size_t ggml_hash(const struct ggml_tensor * p) {
+ // the last 4 bits are always zero due to alignment
+ return (size_t)(uintptr_t)p >> 4;
+}
+
+static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key) {
+ size_t h = ggml_hash(key) % hash_set->size;
+
+ // linear probing
+ size_t i = h;
+ while (ggml_bitset_get(hash_set->used, i) && hash_set->keys[i] != key) {
+ i = (i + 1) % hash_set->size;
+ if (i == h) {
+ // visited all hash table entries -> not found
+ return GGML_HASHSET_FULL;
+ }
+ }
+ return i;
+}
+
+static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key) {
+ size_t i = ggml_hash_find(hash_set, key);
+ return i != GGML_HASHSET_FULL && ggml_bitset_get(hash_set->used, i);
+}
+
+static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {
+ size_t h = ggml_hash(key) % hash_set->size;
+
+ // linear probing
+ size_t i = h;
+ do {
+ if (!ggml_bitset_get(hash_set->used, i)) {
+ ggml_bitset_set(hash_set->used, i);
+ hash_set->keys[i] = key;
+ return i;
+ }
+ if (hash_set->keys[i] == key) {
+ return GGML_HASHSET_ALREADY_EXISTS;
+ }
+ i = (i + 1) % hash_set->size;
+ } while (i != h);
+
+ // visited all hash table entries -> not found
+ GGML_ABORT("fatal error");
+}
+
+static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {
+ size_t h = ggml_hash(key) % hash_set->size;
+
+ // linear probing
+ size_t i = h;
+ do {
+ if (!ggml_bitset_get(hash_set->used, i)) {
+ ggml_bitset_set(hash_set->used, i);
+ hash_set->keys[i] = key;
+ return i;
+ }
+ if (hash_set->keys[i] == key) {
+ return i;
+ }
+ i = (i + 1) % hash_set->size;
+ } while (i != h);
+
+ // visited all hash table entries -> not found
+ GGML_ABORT("fatal error");
+}
+
+// computation graph
+
+enum ggml_cgraph_eval_order {
+ GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
+ GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
+ GGML_CGRAPH_EVAL_ORDER_COUNT
+};
+
+struct ggml_cgraph {
+ int size; // maximum number of nodes/leafs/grads/grad_accs
+ int n_nodes; // number of nodes currently in use
+ int n_leafs; // number of leafs currently in use
+
+ struct ggml_tensor ** nodes; // tensors with data that can change if the graph is evaluated
+ struct ggml_tensor ** grads; // the outputs of these tensors are the gradients of the nodes
+ struct ggml_tensor ** grad_accs; // accumulators for node gradients
+ struct ggml_tensor ** leafs; // tensors with constant data
+ int32_t * use_counts;// number of uses of each tensor, indexed by hash table slot
+
+ struct ggml_hash_set visited_hash_set;
+
+ enum ggml_cgraph_eval_order order;
+
+ // an optional identifier that can be utilized to recognize same graphs if two non-zero values match
+ // a value of 0 means it is not set and should be ignored
+ uint64_t uid;
+};
+
+// returns a slice of cgraph with nodes [i0, i1)
+// the slice does not have leafs or gradients
+// if you need the gradients, get them from the original graph
+struct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph, int i0, int i1);
+
+// ggml-alloc.c: true if the operation can reuse memory from its sources
+GGML_API bool ggml_op_can_inplace(enum ggml_op op);
+
+
+// Memory allocation
+
+GGML_API void * ggml_aligned_malloc(size_t size);
+GGML_API void ggml_aligned_free(void * ptr, size_t size);
+
+// FP16 <-> FP32
+// ref: https://github.com/Maratyszcza/FP16
+
+static inline float fp32_from_bits(uint32_t w) {
+ union {
+ uint32_t as_bits;
+ float as_value;
+ } fp32;
+ fp32.as_bits = w;
+ return fp32.as_value;
+}
+
+static inline uint32_t fp32_to_bits(float f) {
+ union {
+ float as_value;
+ uint32_t as_bits;
+ } fp32;
+ fp32.as_value = f;
+ return fp32.as_bits;
+}
+
+static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
+ const uint32_t w = (uint32_t) h << 16;
+ const uint32_t sign = w & UINT32_C(0x80000000);
+ const uint32_t two_w = w + w;
+
+ const uint32_t exp_offset = UINT32_C(0xE0) << 23;
+#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)
+ const float exp_scale = 0x1.0p-112f;
+#else
+ const float exp_scale = fp32_from_bits(UINT32_C(0x7800000));
+#endif
+ const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale;
+
+ const uint32_t magic_mask = UINT32_C(126) << 23;
+ const float magic_bias = 0.5f;
+ const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias;
+
+ const uint32_t denormalized_cutoff = UINT32_C(1) << 27;
+ const uint32_t result = sign |
+ (two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));
+ return fp32_from_bits(result);
+}
+
+static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {
+#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)
+ const float scale_to_inf = 0x1.0p+112f;
+ const float scale_to_zero = 0x1.0p-110f;
+#else
+ const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000));
+ const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000));
+#endif
+ float base = (fabsf(f) * scale_to_inf) * scale_to_zero;
+
+ const uint32_t w = fp32_to_bits(f);
+ const uint32_t shl1_w = w + w;
+ const uint32_t sign = w & UINT32_C(0x80000000);
+ uint32_t bias = shl1_w & UINT32_C(0xFF000000);
+ if (bias < UINT32_C(0x71000000)) {
+ bias = UINT32_C(0x71000000);
+ }
+
+ base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;
+ const uint32_t bits = fp32_to_bits(base);
+ const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);
+ const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);
+ const uint32_t nonsign = exp_bits + mantissa_bits;
+ return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign);
+}
+
+#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)
+
+#define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x)
+#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)
+
+static inline float ggml_e8m0_to_fp32(uint8_t x) {
+ uint32_t bits; // Stores the raw bit representation of the float
+
+ // Handle special case for minimum exponent (denormalized float)
+ if (x == 0) {
+ // Bit pattern for 2^(-127):
+ // - Sign bit: 0 (positive)
+ // - Exponent: 0 (denormalized number)
+ // - Mantissa: 0x400000 (0.5 in fractional form)
+ // Value = 0.5 * 2^(-126) = 2^(-127)
+ bits = 0x00400000;
+ }
+ // note: disabled as we don't need to handle NaNs
+ //// Handle special case for NaN (all bits set)
+ //else if (x == 0xFF) {
+ // // Standard quiet NaN pattern:
+ // // - Sign bit: 0
+ // // - Exponent: all 1s (0xFF)
+ // // - Mantissa: 0x400000 (quiet NaN flag)
+ // bits = 0x7FC00000;
+ //}
+ // Normalized values (most common case)
+ else {
+ // Construct normalized float by shifting exponent into position:
+ // - Exponent field: 8 bits (positions 30-23)
+ // - Mantissa: 0 (implicit leading 1)
+ // Value = 2^(x - 127)
+ bits = (uint32_t) x << 23;
+ }
+
+ float result; // Final float value
+ // Safely reinterpret bit pattern as float without type-punning issues
+ memcpy(&result, &bits, sizeof(float));
+ return result;
+}
+
+// Equal to ggml_e8m0_to_fp32/2
+// Useful with MXFP4 quantization since the E0M2 values are doubled
+static inline float ggml_e8m0_to_fp32_half(uint8_t x) {
+ uint32_t bits;
+
+ // For x < 2: use precomputed denormal patterns
+ if (x < 2) {
+ // 0x00200000 = 2^(-128), 0x00400000 = 2^(-127)
+ bits = 0x00200000 << x;
+ }
+ // For x >= 2: normalized exponent adjustment
+ else {
+ // 0.5 * 2^(x-127) = 2^(x-128) = normalized with exponent (x-1)
+ bits = (uint32_t)(x - 1) << 23;
+ }
+ // Note: NaNs are not handled here
+
+ float result;
+ memcpy(&result, &bits, sizeof(float));
+ return result;
+}
+
+#define GGML_E8M0_TO_FP32(x) ggml_e8m0_to_fp32(x)
+#define GGML_E8M0_TO_FP32_HALF(x) ggml_e8m0_to_fp32_half(x)
+
+// UE4M3: unsigned, 4 exp bits (bias=7), 3 mantissa bits
+// Returns value * 0.5 to match kvalues_mxfp4 convention (kvalues = 2 * E2M1_float)
+static inline float ggml_ue4m3_to_fp32(uint8_t x) {
+ if (x == 0 || x == 0x7F) {
+ return 0.0f;
+ }
+ int exp = (x >> 3) & 0xF;
+ int man = x & 0x7;
+ float raw;
+ if (exp == 0) {
+ raw = ldexpf((float) man, -9);
+ } else {
+ raw = ldexpf(1.0f + (float) man / 8.0f, exp - 7);
+ }
+ return raw * 0.5f;
+}
+
+static inline uint8_t ggml_fp32_to_ue4m3(float x) {
+ if (!(x > 0.0f)) {
+ return 0;
+ }
+ if (x > 448.0f) {
+ x = 448.0f;
+ }
+ uint32_t bits;
+ memcpy(&bits, &x, 4);
+ int fp32_exp = ((bits >> 23) & 0xFF) - 127;
+ int fp32_man = (bits >> 20) & 0x7;
+ int ue4m3_exp = fp32_exp + 7;
+ if (ue4m3_exp <= 0) {
+ // subnormal: value = man * 2^-9, man = round(x * 2^9)
+ int man = (int) (x * 512.0f + 0.5f);
+ if (man > 7) {
+ man = 7;
+ }
+ if (man < 1) {
+ return 0;
+ }
+ return (uint8_t) man;
+ }
+ if (ue4m3_exp >= 15) {
+ return 0x7E;
+ }
+ int round_bit = (bits >> 19) & 1;
+ int ue4m3_man = fp32_man + round_bit;
+ if (ue4m3_man > 7) {
+ ue4m3_man = 0;
+ ue4m3_exp++;
+ if (ue4m3_exp >= 15) {
+ return 0x7E;
+ }
+ }
+ return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man);
+}
+
+/**
+ * Converts brain16 to float32.
+ *
+ * The bfloat16 floating point format has the following structure:
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌──┴───┐┌─┴───┐
+ * 0b0000000000000000 brain16
+ *
+ * Since bf16 has the same number of exponent bits as a 32bit float,
+ * encoding and decoding numbers becomes relatively straightforward.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌──┴───┐┌─┴───────────────────┐
+ * 0b00000000000000000000000000000000 IEEE binary32
+ *
+ * For comparison, the standard fp16 format has fewer exponent bits.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌─┴─┐┌─┴──────┐
+ * 0b0000000000000000 IEEE binary16
+ *
+ * @see IEEE 754-2008
+ */
+static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {
+ union {
+ float f;
+ uint32_t i;
+ } u;
+ u.i = (uint32_t)h.bits << 16;
+ return u.f;
+}
+
+/**
+ * Converts float32 to brain16.
+ *
+ * This is binary identical with Google Brain float conversion.
+ * Floats shall round to nearest even, and NANs shall be quiet.
+ * Subnormals aren't flushed to zero, except perhaps when used.
+ * This code should vectorize nicely if using modern compilers.
+ */
+static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {
+ ggml_bf16_t h;
+ union {
+ float f;
+ uint32_t i;
+ } u;
+ u.f = s;
+ if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */
+ h.bits = (u.i >> 16) | 64; /* force to quiet */
+ return h;
+ }
+ h.bits = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;
+ return h;
+}
+
+#define GGML_FP32_TO_BF16(x) ggml_compute_fp32_to_bf16(x)
+#define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x)
+
+static inline int32_t ggml_node_get_use_count(const struct ggml_cgraph * cgraph, int node_idx) {
+ const struct ggml_tensor * node = cgraph->nodes[node_idx];
+
+ size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, node);
+ if (!ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) {
+ return 0;
+ }
+ return cgraph->use_counts[hash_pos];
+}
+
+// return true if the node's results are only used by N other nodes
+// and can be fused into their calculations.
+static inline bool ggml_node_has_n_uses(const struct ggml_cgraph * cgraph, int node_idx, int32_t n_uses) {
+ const struct ggml_tensor * node = cgraph->nodes[node_idx];
+
+ // check the use count against how many we're replacing
+ if (ggml_node_get_use_count(cgraph, node_idx) != n_uses) {
+ return false;
+ }
+
+ // if node is a view, some other node might be using the intermediate result
+ // via the view source.
+ if (node->view_src) {
+ return false;
+ }
+
+ // If the user requested output for the node, can't fuse
+ if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {
+ return false;
+ }
+
+ return true;
+}
+
+// Returns true if nodes with indices { node_idxs } are the sequence of ggml_ops in ops[]
+// and are fusable. Nodes are considered fusable according to this function if:
+// - all nodes except the last have only one use and are not views/outputs (see ggml_node_has_N_uses).
+// - all nodes except the last are a src of the following node.
+// - all nodes are the same shape.
+// TODO: Consider allowing GGML_OP_NONE nodes in between
+static inline bool ggml_can_fuse_ext(const struct ggml_cgraph * cgraph, const int * node_idxs, const enum ggml_op * ops, int num_ops) {
+ for (int i = 0; i < num_ops; ++i) {
+ if (node_idxs[i] >= cgraph->n_nodes) {
+ return false;
+ }
+
+ struct ggml_tensor * node = cgraph->nodes[node_idxs[i]];
+ if (node->op != ops[i]) {
+ return false;
+ }
+ if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
+ return false;
+ }
+ if (i < num_ops - 1 && !ggml_node_has_n_uses(cgraph, node_idxs[i], 1)) {
+ return false;
+ }
+ if (i > 0) {
+ struct ggml_tensor * prev = cgraph->nodes[node_idxs[i - 1]];
+ if (node->src[0] != prev && node->src[1] != prev) {
+ return false;
+ }
+ if (!ggml_are_same_shape(node, prev)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+// same as above, for sequential indices starting at node_idx
+static inline bool ggml_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, const enum ggml_op * ops, int num_ops) {
+ assert(num_ops < 32);
+
+ if (node_idx + num_ops > cgraph->n_nodes) {
+ return false;
+ }
+
+ int idxs[32];
+ for (int i = 0; i < num_ops; ++i) {
+ idxs[i] = node_idx + i;
+ }
+
+ return ggml_can_fuse_ext(cgraph, idxs, ops, num_ops);
+}
+
+GGML_API bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph,
+ const int * node_idxs,
+ int count,
+ const enum ggml_op * ops,
+ const int * outputs,
+ int num_outputs);
+
+// Returns true if the subgraph formed by {node_idxs} can be fused
+// checks whethers all nodes which are not part of outputs can be elided
+// by checking if their num_uses are confined to the subgraph
+static inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph,
+ int node_idx,
+ int count,
+ const enum ggml_op * ops,
+ const int * outputs,
+ int num_outputs) {
+ GGML_ASSERT(count < 32);
+ if (node_idx + count > cgraph->n_nodes) {
+ return false;
+ }
+
+ int idxs[32];
+
+ for (int i = 0; i < count; ++i) {
+ idxs[i] = node_idx + i;
+ }
+
+ return ggml_can_fuse_subgraph_ext(cgraph, idxs, count, ops, outputs, num_outputs);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+#include <array>
+#include <initializer_list>
+#include <vector>
+
+// nicer C++ syntax for ggml_can_fuse
+inline bool ggml_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {
+ return ggml_can_fuse(cgraph, node_idx, ops.begin(), (int)ops.size());
+}
+
+inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph,
+ int start_idx,
+ std::initializer_list<enum ggml_op> ops,
+ std::initializer_list<int> outputs = {}) {
+ return ggml_can_fuse_subgraph(cgraph, start_idx, ops.size(), ops.begin(), outputs.begin(), outputs.size());
+}
+
+// Return true if the edges in the graph match expectations.
+inline bool ggml_check_edges(const struct ggml_cgraph * cgraph,
+ int start_idx,
+ std::initializer_list<std::array<int, 3>> edges) {
+ for (const auto & edge : edges) {
+ int dst_node = edge[0];
+ int src_idx = edge[1];
+ int src_node = edge[2];
+ if (cgraph->nodes[start_idx + dst_node]->src[src_idx] != cgraph->nodes[start_idx + src_node]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// expose GGUF internals for test code
+GGML_API size_t gguf_type_size(enum gguf_type type);
+GGML_API void gguf_write_to_buf(const struct gguf_context * ctx, std::vector<int8_t> & buf, bool only_meta);
+#endif // __cplusplus
diff --git a/python/freetoken/kernel/csrc/gguf/dequantize.cuh b/python/freetoken/kernel/csrc/gguf/dequantize.cuh
deleted file mode 100644
index 57cb1a27..00000000
--- a/python/freetoken/kernel/csrc/gguf/dequantize.cuh
+++ /dev/null
@@ -1,583 +0,0 @@
-// copied from
-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/dequantize.cuh
-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu
-// Dequant functions
-static __device__ __forceinline__ void dequantize_q4_0(const void* vx, const int ib, const int iqs, dfloat2& v) {
- const block_q4_0* x = (const block_q4_0*)vx;
-
- const dfloat d = x[ib].d;
-
- const int vui = x[ib].qs[iqs];
-
- v.x = __int2half_rn(vui & 0xF);
- v.y = __int2half_rn(vui >> 4);
-
- v = __hsub2(v, __floats2half2_rn(8.0f, 8.0f));
- v = __hmul2(v, {d, d});
-}
-
-static __device__ __forceinline__ void dequantize_q4_1(const void* vx, const int ib, const int iqs, dfloat2& v) {
- const block_q4_1* x = (const block_q4_1*)vx;
-
- const dfloat d = __low2half(x[ib].dm);
- const dfloat m = __high2half(x[ib].dm);
-
- const int vui = x[ib].qs[iqs];
-
- v.x = __int2half_rn(vui & 0xF);
- v.y = __int2half_rn(vui >> 4);
-
- v = __hmul2(v, {d, d});
- v = __hadd2(v, {m, m});
-}
-
-static __device__ __forceinline__ void dequantize_q5_0(const void* vx, const int ib, const int iqs, dfloat2& v) {
- const block_q5_0* x = (const block_q5_0*)vx;
-
- const dfloat d = x[ib].d;
-
- uint32_t qh;
- memcpy(&qh, x[ib].qh, sizeof(qh));
-
- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
- const int xh_1 = ((qh >> (iqs + 12))) & 0x10;
-
- v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
- v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
-
- v = __hsub2(v, __floats2half2_rn(16.0f, 16.0f));
- v = __hmul2(v, {d, d});
-}
-
-static __device__ __forceinline__ void dequantize_q5_1(const void* vx, const int ib, const int iqs, dfloat2& v) {
- const block_q5_1* x = (const block_q5_1*)vx;
-
- const dfloat d = __low2half(x[ib].dm);
- const dfloat m = __high2half(x[ib].dm);
-
- uint32_t qh;
- memcpy(&qh, x[ib].qh, sizeof(qh));
-
- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
- const int xh_1 = ((qh >> (iqs + 12))) & 0x10;
-
- v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
- v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
-
- v = __hmul2(v, {d, d});
- v = __hadd2(v, {m, m});
-}
-
-static __device__ __forceinline__ void dequantize_q8_0(const void* vx, const int ib, const int iqs, dfloat2& v) {
- const block_q8_0* x = (const block_q8_0*)vx;
-
- const dfloat d = x[ib].d;
-
- v.x = __int2half_rn(x[ib].qs[iqs + 0]);
- v.y = __int2half_rn(x[ib].qs[iqs + 1]);
-
- v = __hmul2(v, {d, d});
-}
-
-template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
-static __global__ void dequantize_block(const void* __restrict__ vx, dst_t* __restrict__ y, const int k) {
- const int i = 2 * (blockDim.x * blockIdx.x + threadIdx.x);
-
- if (i >= k) {
- return;
- }
-
- const int ib = i / qk; // block index
- const int iqs = (i % qk) / qr; // quant index
- const int iybs = i - i % qk; // y block start index
- const int y_offset = qr == 1 ? 1 : qk / 2;
-
- // dequantize
- dfloat2 v;
- dequantize_kernel(vx, ib, iqs, v);
-
- y[iybs + iqs + 0] = convert_from_half<dst_t>(v.x);
- y[iybs + iqs + y_offset] = convert_from_half<dst_t>(v.y);
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_q2_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_q2_K* x = (const block_q2_K*)vx;
-
- const auto tid = threadIdx.x;
- const int n = tid / 32;
- const int l = tid - 32 * n;
- const int is = 8 * n + l / 16;
-
- const uint8_t q = x[i].qs[32 * n + l];
- dst_t* y = yy + i * QK_K + 128 * n;
-
- half dall = __low2half(x[i].dm);
- half dmin = __high2half(x[i].dm);
- y[l + 0] = convert_from_half<dst_t>(__hsub(
- __hmul(dall, __int2half_rn((x[i].scales[is + 0] & 0xF) * ((q >> 0) & 3))),
- __hmul(dmin, __int2half_rn(x[i].scales[is + 0] >> 4))));
- y[l + 32] = convert_from_half<dst_t>(__hsub(
- __hmul(dall, __int2half_rn((x[i].scales[is + 2] & 0xF) * ((q >> 2) & 3))),
- __hmul(dmin, __int2half_rn(x[i].scales[is + 2] >> 4))));
- y[l + 64] = convert_from_half<dst_t>(__hsub(
- __hmul(dall, __int2half_rn((x[i].scales[is + 4] & 0xF) * ((q >> 4) & 3))),
- __hmul(dmin, __int2half_rn(x[i].scales[is + 4] >> 4))));
- y[l + 96] = convert_from_half<dst_t>(__hsub(
- __hmul(dall, __int2half_rn((x[i].scales[is + 6] & 0xF) * ((q >> 6) & 3))),
- __hmul(dmin, __int2half_rn(x[i].scales[is + 6] >> 4))));
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_q3_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_q3_K* x = (const block_q3_K*)vx;
-
- const auto r = threadIdx.x / 4;
- const int tid = r / 2;
- const int is0 = r % 2;
- const int l0 = 16 * is0 + 4 * (threadIdx.x % 4);
- const int n = tid / 4;
- const int j = tid - 4 * n;
-
- uint8_t m = 1 << (4 * n + j);
- int is = 8 * n + 2 * j + is0;
- int shift = 2 * j;
-
- int8_t us = is < 4 ? (x[i].scales[is - 0] & 0xF) | (((x[i].scales[is + 8] >> 0) & 3) << 4)
- : is < 8 ? (x[i].scales[is - 0] & 0xF) | (((x[i].scales[is + 4] >> 2) & 3) << 4)
- : is < 12 ? (x[i].scales[is - 8] >> 4) | (((x[i].scales[is + 0] >> 4) & 3) << 4)
- : (x[i].scales[is - 8] >> 4) | (((x[i].scales[is - 4] >> 6) & 3) << 4);
- half d_all = x[i].d;
- half dl = __hmul(d_all, __int2half_rn(us - 32));
-
- dst_t* y = yy + i * QK_K + 128 * n + 32 * j;
- const uint8_t* q = x[i].qs + 32 * n;
- const uint8_t* hm = x[i].hmask;
-
- for (int l = l0; l < l0 + 4; ++l) {
- y[l] = convert_from_half<dst_t>(__hmul(dl, __int2half_rn((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4))));
- }
-}
-
-static inline __device__ void get_scale_min_k4(int j, const uint8_t* q, uint8_t& d, uint8_t& m) {
- if (j < 4) {
- d = q[j] & 63;
- m = q[j + 4] & 63;
- } else {
- d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
- m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);
- }
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_q4_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const block_q4_K* x = (const block_q4_K*)vx;
-
- const auto i = blockIdx.x;
-
- // assume 32 threads
- const auto tid = threadIdx.x;
- const int il = tid / 8;
- const int ir = tid % 8;
- const int is = 2 * il;
- const int n = 4;
-
- dst_t* y = yy + i * QK_K + 64 * il + n * ir;
-
- const half dall = __low2half(x[i].dm);
- const half dmin = __high2half(x[i].dm);
-
- const uint8_t* q = x[i].qs + 32 * il + n * ir;
-
- uint8_t sc, m;
- get_scale_min_k4(is + 0, x[i].scales, sc, m);
- const half d1 = __hmul(dall, __int2half_rn(sc));
- const half m1 = __hmul(dmin, __int2half_rn(m));
- get_scale_min_k4(is + 1, x[i].scales, sc, m);
- const half d2 = __hmul(dall, __int2half_rn(sc));
- const half m2 = __hmul(dmin, __int2half_rn(m));
- for (int l = 0; l < n; ++l) {
- y[l + 0] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn(q[l] & 0xF)), m1));
- y[l + 32] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn(q[l] >> 4)), m2));
- }
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_q5_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const block_q5_K* x = (const block_q5_K*)vx;
-
- const auto i = blockIdx.x;
-
- // assume 64 threads - this is very slightly better than the one below
- const auto tid = threadIdx.x;
- const int il = tid / 16; // il is in 0...3
- const int ir = tid % 16; // ir is in 0...15
- const int is = 2 * il; // is is in 0...6
-
- dst_t* y = yy + i * QK_K + 64 * il + 2 * ir;
-
- const half dall = __low2half(x[i].dm);
- const half dmin = __high2half(x[i].dm);
-
- const uint8_t* ql = x[i].qs + 32 * il + 2 * ir;
- const uint8_t* qh = x[i].qh + 2 * ir;
-
- uint8_t sc, m;
- get_scale_min_k4(is + 0, x[i].scales, sc, m);
- const half d1 = __hmul(dall, __int2half_rn(sc));
- const half m1 = __hmul(dmin, __int2half_rn(m));
- get_scale_min_k4(is + 1, x[i].scales, sc, m);
- const half d2 = __hmul(dall, __int2half_rn(sc));
- const half m2 = __hmul(dmin, __int2half_rn(m));
-
- uint8_t hm = 1 << (2 * il);
- y[0] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn((ql[0] & 0xF) + (qh[0] & hm ? 16 : 0))), m1));
- y[1] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn((ql[1] & 0xF) + (qh[1] & hm ? 16 : 0))), m1));
- hm <<= 1;
- y[32] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn((ql[0] >> 4) + (qh[0] & hm ? 16 : 0))), m2));
- y[33] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn((ql[1] >> 4) + (qh[1] & hm ? 16 : 0))), m2));
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_q6_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const block_q6_K* x = (const block_q6_K*)vx;
-
- const auto i = blockIdx.x;
-
- // assume 64 threads - this is very slightly better than the one below
- const auto tid = threadIdx.x;
- const int ip = tid / 32; // ip is 0 or 1
- const int il = tid - 32 * ip; // 0...32
- const int is = 8 * ip + il / 16;
-
- dst_t* y = yy + i * QK_K + 128 * ip + il;
-
- const half d = x[i].d;
-
- const uint8_t* ql = x[i].ql + 64 * ip + il;
- const uint8_t qh = x[i].qh[32 * ip + il];
- const int8_t* sc = x[i].scales + is;
-
- y[0] = convert_from_half<dst_t>(
- __hmul(d, __int2half_rn(sc[0] * ((int8_t)((ql[0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32))));
- y[32] = convert_from_half<dst_t>(
- __hmul(d, __int2half_rn(sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32))));
- y[64] = convert_from_half<dst_t>(
- __hmul(d, __int2half_rn(sc[4] * ((int8_t)((ql[0] >> 4) | (((qh >> 4) & 3) << 4)) - 32))));
- y[96] = convert_from_half<dst_t>(
- __hmul(d, __int2half_rn(sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32))));
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq2_xxs(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_iq2_xxs* x = (const block_iq2_xxs*)vx;
-
- const auto tid = threadIdx.x;
- const int il = tid / 8; // 0...3
- const int ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;
- const uint16_t* q2 = x[i].qs + 4 * ib;
- const uint8_t* aux8 = (const uint8_t*)q2;
- const uint8_t* grid = (const uint8_t*)(iq2xxs_grid + aux8[il]);
- const uint32_t aux32 = q2[2] | (q2[3] << 16);
- const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.25f;
- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7 * il) & 127];
- for (int j = 0; j < 8; ++j)
- y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq2_xs(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_iq2_xs* x = (const block_iq2_xs*)vx;
-
- const auto tid = threadIdx.x;
- const int il = tid / 8; // 0...3
- const int ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;
- const uint16_t* q2 = x[i].qs + 4 * ib;
- const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[il] & 511));
- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4 * (il / 2)) & 0xf)) * 0.25f;
- const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];
- for (int j = 0; j < 8; ++j)
- y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq2_s(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_iq2_s* x = (const block_iq2_s*)vx;
-
- const auto tid = threadIdx.x;
- const int il = tid / 8; // 0...3
- const int ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;
- const uint8_t* grid = (const uint8_t*)(iq2s_grid + (x[i].qs[4 * ib + il] | ((x[i].qh[ib] << (8 - 2 * il)) & 0x300)));
- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4 * (il / 2)) & 0xf)) * 0.25f;
- const uint8_t signs = x[i].qs[QK_K / 8 + 4 * ib + il];
- for (int j = 0; j < 8; ++j)
- y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq3_xxs(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_iq3_xxs* x = (const block_iq3_xxs*)vx;
-
- const auto tid = threadIdx.x;
- const int il = tid / 8; // 0...3
- const int ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;
- const uint8_t* q3 = x[i].qs + 8 * ib;
- const uint16_t* gas = (const uint16_t*)(x[i].qs + QK_K / 4) + 2 * ib;
- const uint8_t* grid1 = (const uint8_t*)(iq3xxs_grid + q3[2 * il + 0]);
- const uint8_t* grid2 = (const uint8_t*)(iq3xxs_grid + q3[2 * il + 1]);
- const uint32_t aux32 = gas[0] | (gas[1] << 16);
- const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.5f;
- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7 * il) & 127];
- for (int j = 0; j < 4; ++j) {
- y[j + 0] = d * grid1[j] * (signs & kmask_iq2xs[j + 0] ? -1.f : 1.f);
- y[j + 4] = d * grid2[j] * (signs & kmask_iq2xs[j + 4] ? -1.f : 1.f);
- }
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq3_s(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_iq3_s* x = (const block_iq3_s*)vx;
-
- const auto tid = threadIdx.x;
- const int il = tid / 8; // 0...3
- const int ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;
- const uint8_t* qs = x[i].qs + 8 * ib;
- const uint8_t* grid1 = (const uint8_t*)(iq3xs_grid + (qs[2 * il + 0] | ((x[i].qh[ib] << (8 - 2 * il)) & 256)));
- const uint8_t* grid2 = (const uint8_t*)(iq3xs_grid + (qs[2 * il + 1] | ((x[i].qh[ib] << (7 - 2 * il)) & 256)));
- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib / 2] >> 4 * (ib % 2)) & 0xf)) * 0.5f;
- const uint8_t signs = x[i].signs[4 * ib + il];
- for (int j = 0; j < 4; ++j) {
- y[j + 0] = d * grid1[j] * (signs & kmask_iq2xs[j + 0] ? -1.f : 1.f);
- y[j + 4] = d * grid2[j] * (signs & kmask_iq2xs[j + 4] ? -1.f : 1.f);
- }
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq1_s(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const int64_t i = blockIdx.x;
- const block_iq1_s* x = (const block_iq1_s*)vx;
-
- const int64_t tid = threadIdx.x;
- const int64_t il = tid / 8; // 0...3
- const int64_t ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;
- const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;
- const float d = __half2float(x[i].d) * (2 * ((x[i].qh[ib] >> 12) & 7) + 1);
- uint32_t grid32[2];
- const int8_t* q = (const int8_t*)grid32;
- grid32[0] = iq1s_grid_gpu[x[i].qs[4 * ib + il] | (((x[i].qh[ib] >> 3 * il) & 7) << 8)];
- grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
- grid32[0] &= 0x0f0f0f0f;
- for (int j = 0; j < 8; ++j) {
- y[j] = d * (q[j] + delta);
- }
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq1_m(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const int64_t i = blockIdx.x;
- const block_iq1_m* x = (const block_iq1_m*)vx;
-
- const int64_t tid = threadIdx.x;
- const int64_t il = tid / 8; // 0...3
- const int64_t ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;
- const uint16_t* sc = (const uint16_t*)x[i].scales;
- iq1m_scale_t scale;
- scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
- const int64_t ib16 = 2 * ib + il / 2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);
- const float d = __half2float(scale.f16) * (2 * ((sc[ib16 / 4] >> 3 * (ib16 % 4)) & 0x7) + 1);
- const float delta = x[i].qh[2 * ib + il / 2] & (0x08 << 4 * (il % 2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;
- uint32_t grid32[2];
- const int8_t* q = (const int8_t*)grid32;
- grid32[0] = iq1s_grid_gpu[x[i].qs[4 * ib + il] | (((x[i].qh[2 * ib + il / 2] >> 4 * (il % 2)) & 7) << 8)];
- grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
- grid32[0] &= 0x0f0f0f0f;
- for (int j = 0; j < 8; ++j) {
- y[j] = d * (q[j] + delta);
- }
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq4_nl(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_iq4_nl* x = (const block_iq4_nl*)vx + i * (QK_K / QK4_NL);
-
- const auto tid = threadIdx.x;
- const int il = tid / 8; // 0...3
- const int ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 4 * il;
- const uint8_t* q4 = x[ib].qs + 4 * il;
- const float d = __half2float(x[ib].d);
- for (int j = 0; j < 4; ++j) {
- y[j + 0] = d * kvalues_iq4nl[q4[j] & 0xf];
- y[j + 16] = d * kvalues_iq4nl[q4[j] >> 4];
- }
-}
-
-template <typename dst_t>
-static __global__ void dequantize_block_iq4_xs(const void* __restrict__ vx, dst_t* __restrict__ yy) {
- const auto i = blockIdx.x;
- const block_iq4_xs* x = (const block_iq4_xs*)vx;
-
- const auto tid = threadIdx.x;
- const int il = tid / 8; // 0...3
- const int ib = tid % 8; // 0...7
- dst_t* y = yy + i * QK_K + 32 * ib + 4 * il;
- const uint8_t* q4 = x[i].qs + 16 * ib + 4 * il;
- const float d = __half2float(x[i].d) *
- ((((x[i].scales_l[ib / 2] >> 4 * (ib % 2)) & 0xf) | (((x[i].scales_h >> 2 * ib) & 3) << 4)) - 32);
- for (int j = 0; j < 4; ++j) {
- y[j + 0] = d * kvalues_iq4nl[q4[j] & 0xf];
- y[j + 16] = d * kvalues_iq4nl[q4[j] >> 4];
- }
-}
-
-template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
-static void
-dequantize_block_cuda(const void* __restrict__ vx, dst_t* __restrict__ y, const int k, cudaStream_t stream) {
- const int num_blocks = (k + 2 * CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2 * CUDA_DEQUANTIZE_BLOCK_SIZE);
- dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);
-}
-
-template <typename dst_t>
-static void dequantize_row_q2_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_q3_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_q4_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_q5_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_q6_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq2_xxs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq2_xs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq2_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq3_xxs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq3_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq1_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq1_m_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = k / QK_K;
- dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq4_nl_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = (k + QK_K - 1) / QK_K;
- dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static void dequantize_row_iq4_xs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {
- const int nb = (k + QK_K - 1) / QK_K;
- dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
-}
-
-template <typename dst_t>
-static to_cuda_ggml_t<dst_t> ggml_get_to_cuda(int64_t type) {
- switch (type) {
- case 2:
- return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
- case 3:
- return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;
- case 6:
- return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
- case 7:
- return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
- case 8:
- return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
- case 10:
- return dequantize_row_q2_K_cuda;
- case 11:
- return dequantize_row_q3_K_cuda;
- case 12:
- return dequantize_row_q4_K_cuda;
- case 13:
- return dequantize_row_q5_K_cuda;
- case 14:
- return dequantize_row_q6_K_cuda;
- case 16:
- return dequantize_row_iq2_xxs_cuda;
- case 17:
- return dequantize_row_iq2_xs_cuda;
- case 18:
- return dequantize_row_iq3_xxs_cuda;
- case 19:
- return dequantize_row_iq1_s_cuda;
- case 20:
- return dequantize_row_iq4_nl_cuda;
- case 21:
- return dequantize_row_iq3_s_cuda;
- case 22:
- return dequantize_row_iq2_s_cuda;
- case 23:
- return dequantize_row_iq4_xs_cuda;
- case 29:
- return dequantize_row_iq1_m_cuda;
- default:
- return nullptr;
- }
-}
diff --git a/python/freetoken/kernel/csrc/gguf/dispatch.h b/python/freetoken/kernel/csrc/gguf/dispatch.h
deleted file mode 100644
index f42a2163..00000000
--- a/python/freetoken/kernel/csrc/gguf/dispatch.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Minimal AT_DISPATCH helper for the vendored GGUF kernels (borrowed from
-// sgl-kernel csrc/quantization/gguf, which are ports of llama.cpp). The donor
-// pulls these macros from its large include/utils.h; we only need the float
-// dispatch, so vendor just that to keep the JIT compile self-contained.
-#pragma once
-
-#include <ATen/Dispatch.h>
-
-#ifndef WARP_SIZE
-#define WARP_SIZE 32
-#endif
-
-// Warp-shuffle wrappers the donor pulls from sgl-kernel's utils.h (CUDA variants).
-#ifndef SGLANG_SHFL_XOR_SYNC
-#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask))
-#endif
-#ifndef SGLANG_SHFL_XOR_SYNC_WIDTH
-#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) \
- __shfl_xor_sync((mask), (var), (lane_mask), (width))
-#endif
-
-#define DISPATCH_CASE_FLOAT_TYPES(...) \
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
- AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
- AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
-
-#define DISPATCH_FLOAT_TYPES(TYPE, NAME, ...) \
- AT_DISPATCH_SWITCH(TYPE, NAME, DISPATCH_CASE_FLOAT_TYPES(__VA_ARGS__))
diff --git a/python/freetoken/kernel/csrc/gguf/ggml-common.h b/python/freetoken/kernel/csrc/gguf/ggml-common.h
deleted file mode 100644
index 88c21a4a..00000000
--- a/python/freetoken/kernel/csrc/gguf/ggml-common.h
+++ /dev/null
@@ -1,1029 +0,0 @@
-// adapted from
-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/ggml-common.h
-// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h
-#define QK_K 256
-#define K_QUANTS_PER_ITERATION 2
-#define WARP_SIZE_GGUF 32
-#define K_SCALE_SIZE 12
-#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
-#define CUDA_QUANTIZE_BLOCK_SIZE 256
-#define GGML_CUDA_DMMV_X 32
-#define GGML_CUDA_MMV_Y 1
-
-// Data Structures
-// QK = number of values after dequantization
-// QR = QK / number of values before dequantization
-// QI = number of 32 bit integers before dequantization
-
-#define QK4_0 32
-#define QR4_0 2
-#define QI4_0 (QK4_0 / (4 * QR4_0))
-typedef struct {
- half d; // delta
- uint8_t qs[QK4_0 / 2]; // nibbles / quants
-} block_q4_0;
-
-#define QK4_1 32
-#define QR4_1 2
-#define QI4_1 (QK4_1 / (4 * QR4_1))
-typedef struct {
- half2 dm; // dm.x = delta, dm.y = min
- uint8_t qs[QK4_1 / 2]; // nibbles / quants
-} block_q4_1;
-
-#define QK5_0 32
-#define QR5_0 2
-#define QI5_0 (QK5_0 / (4 * QR5_0))
-typedef struct {
- half d; // delta
- uint8_t qh[4]; // 5-th bit of quants
- uint8_t qs[QK5_0 / 2]; // nibbles / quants
-} block_q5_0;
-
-#define QK5_1 32
-#define QR5_1 2
-#define QI5_1 (QK5_1 / (4 * QR5_1))
-typedef struct {
- half2 dm; // dm.x = delta, dm.y = min
- uint8_t qh[4]; // 5-th bit of quants
- uint8_t qs[QK5_1 / 2]; // nibbles / quants
-} block_q5_1;
-
-#define QK8_0 32
-#define QR8_0 1
-#define QI8_0 (QK8_0 / (4 * QR8_0))
-typedef struct {
- half d; // delta
- int8_t qs[QK8_0]; // quants
-} block_q8_0;
-
-#define QK8_1 32
-#define QR8_1 1
-#define QI8_1 (QK8_1 / (4 * QR8_1))
-typedef struct {
- half2 ds; // ds.x = delta, ds.y = sum
- int8_t qs[QK8_0]; // quants
-} block_q8_1;
-
-#define QR2_K 4
-#define QI2_K (QK_K / (4 * QR2_K))
-typedef struct {
- uint8_t scales[QK_K / 16]; // scales and mins, quantized with 4 bits
- uint8_t qs[QK_K / 4]; // quants
- half2 dm; // super-block scale for quantized scales/mins
-} block_q2_K;
-
-#define QR3_K 4
-#define QI3_K (QK_K / (4 * QR3_K))
-typedef struct {
- uint8_t hmask[QK_K / 8]; // quants - high bit
- uint8_t qs[QK_K / 4]; // quants - low 2 bits
- uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits
- half d; // super-block scale
-} block_q3_K;
-
-#define QR4_K 2
-#define QI4_K (QK_K / (4 * QR4_K))
-typedef struct {
- half2 dm; // super-block scale for quantized scales/mins
- uint8_t scales[3 * QK_K / 64]; // scales, quantized with 6 bits
- uint8_t qs[QK_K / 2]; // 4--bit quants
-} block_q4_K;
-
-#define QR5_K 2
-#define QI5_K (QK_K / (4 * QR5_K))
-typedef struct {
- half2 dm; // super-block scale for quantized scales/mins
- uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
- uint8_t qh[QK_K / 8]; // quants, high bit
- uint8_t qs[QK_K / 2]; // quants, low 4 bits
-} block_q5_K;
-
-#define QR6_K 2
-#define QI6_K (QK_K / (4 * QR6_K))
-typedef struct {
- uint8_t ql[QK_K / 2]; // quants, lower 4 bits
- uint8_t qh[QK_K / 4]; // quants, upper 2 bits
- int8_t scales[QK_K / 16]; // scales
- half d; // delta
-} block_q6_K;
-
-#define QR2_XXS 8
-#define QI2_XXS (QK_K / (4 * QR2_XXS))
-typedef struct {
- half d;
- uint16_t qs[QK_K / 8];
-} block_iq2_xxs;
-
-#define QR2_XS 8
-#define QI2_XS (QK_K / (4 * QR2_XS))
-typedef struct {
- half d;
- uint16_t qs[QK_K / 8];
- uint8_t scales[QK_K / 32];
-} block_iq2_xs;
-
-#define QR2_S 8
-#define QI2_S (QK_K / (4 * QR2_S))
-typedef struct {
- half d;
- uint8_t qs[QK_K / 4];
- uint8_t qh[QK_K / 32];
- uint8_t scales[QK_K / 32];
-} block_iq2_s;
-
-#define QR3_XXS 8
-#define QI3_XXS (QK_K / (4 * QR3_XXS))
-typedef struct {
- half d;
- uint8_t qs[3 * (QK_K / 8)];
-} block_iq3_xxs;
-
-#define QR3_XS 8
-#define QI3_XS (QK_K / (4 * QR3_XS))
-#define IQ3S_N_SCALE QK_K / 64
-typedef struct {
- half d;
- uint8_t qs[QK_K / 4];
- uint8_t qh[QK_K / 32];
- uint8_t signs[QK_K / 8];
- uint8_t scales[IQ3S_N_SCALE];
-} block_iq3_s;
-
-// 1.5625 bpw
-#define QR1_S 8
-#define QI1_S (QK_K / (4 * QR1_S))
-typedef struct {
- half d;
- uint8_t qs[QK_K / 8];
- uint16_t qh[QK_K / 32];
-} block_iq1_s;
-
-// 1.75 bpw
-#define QR1_M 8
-#define QI1_M (QK_K / (4 * QR1_M))
-typedef struct {
- uint8_t qs[QK_K / 8]; // grid index, low 8 bits
- uint8_t qh[QK_K / 16]; // grid index, high 3 bits + grid shift bit (for two groups of 8)
- uint8_t scales[QK_K / 32]; // 3-bit block scales (4-bit if QK_K == 64)
-} block_iq1_m;
-
-// Used by IQ1_M quants
-typedef union {
- half f16;
- uint16_t u16;
-} iq1m_scale_t;
-
-#define QK4_NL 32
-#define QR4_NL 2
-#define QI4_NL (QK4_NL / (4 * QR4_NL))
-typedef struct {
- half d;
- uint8_t qs[QK4_NL / 2];
-} block_iq4_nl;
-
-#define QR4_XS 8
-#define QI4_XS (QK_K / (4 * QR4_XS))
-typedef struct {
- half d;
- uint16_t scales_h;
- uint8_t scales_l[QK_K / 64];
- uint8_t qs[QK_K / 2];
-} block_iq4_xs;
-
-static const __device__ uint64_t iq2xxs_grid[256] = {
- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b,
- 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b2b08,
- 0x08080808082b2b2b, 0x0808080819080819, 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08,
- 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,
- 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, 0x0808081908191919,
- 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, 0x0808082b08080808, 0x0808082b0808082b,
- 0x0808082b082b082b, 0x0808082b2b08082b, 0x0808190808080819, 0x0808190808081908, 0x0808190808190808,
- 0x08081908082b0819, 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,
- 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x080819082b2b1908,
- 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, 0x08081919082b0808, 0x080819191908192b,
- 0x08081919192b2b19, 0x080819192b080808, 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808,
- 0x0808192b19080808, 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,
- 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, 0x08082b0819081908,
- 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, 0x08082b1908081908, 0x08082b1919080808,
- 0x08082b2b0808082b, 0x08082b2b08191908, 0x0819080808080819, 0x0819080808081908, 0x0819080808190808,
- 0x08190808082b0819, 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,
- 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, 0x0819081919190808,
- 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, 0x0819082b19081919, 0x0819190808080808,
- 0x0819190808082b08, 0x08191908082b0808, 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808,
- 0x0819191908192b08, 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,
- 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, 0x08192b1908080808,
- 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, 0x082b080808080808, 0x082b08080808082b,
- 0x082b080808082b2b, 0x082b080819081908, 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b,
- 0x082b0819082b2b19, 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,
- 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, 0x082b191908080808,
- 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, 0x082b2b0808082b08, 0x082b2b08082b0808,
- 0x082b2b082b191908, 0x082b2b2b19081908, 0x1908080808080819, 0x1908080808081908, 0x1908080808190808,
- 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,
- 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,
- 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, 0x190808192b080808, 0x190808192b081919,
- 0x1908082b08080819, 0x1908082b08190808, 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08,
- 0x1908190808080808, 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,
- 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, 0x19082b0808081908,
- 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b1908080808, 0x19082b1919192b08,
- 0x19082b19192b0819, 0x19082b192b08082b, 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808,
- 0x1919080808082b08, 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,
- 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, 0x1919082b2b190819,
- 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, 0x1919192b08080819, 0x1919192b19191908,
- 0x19192b0808080808, 0x19192b0808190819, 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808,
- 0x19192b2b08082b08, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,
- 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, 0x192b190808080808,
- 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, 0x192b19192b081908, 0x192b2b081908082b,
- 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b,
- 0x2b08081908081908, 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,
- 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808,
- 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, 0x2b08192b08082b19, 0x2b08192b19080808,
- 0x2b08192b192b0808, 0x2b082b080808082b, 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908,
- 0x2b19080808190808, 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,
- 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, 0x2b19190819081908,
- 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, 0x2b2b08080808082b, 0x2b2b080819190808,
- 0x2b2b08082b081919, 0x2b2b081908082b19, 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808,
- 0x2b2b2b1908081908,
-};
-
-static const __device__ uint64_t iq2xs_grid[512] = {
- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b,
- 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, 0x0808080808192b19, 0x08080808082b0808,
- 0x08080808082b082b, 0x08080808082b1919, 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908,
- 0x080808081908192b, 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b,
- 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b192b19,
- 0x080808082b2b0808, 0x0808081908080819, 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19,
- 0x0808081908190808, 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,
- 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, 0x0808081919081919,
- 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, 0x08080819192b0808, 0x08080819192b2b08,
- 0x080808192b080819, 0x080808192b081908, 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b,
- 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,
- 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, 0x0808082b2b080808,
- 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, 0x0808190808082b19,
- 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819,
- 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,
- 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, 0x080819082b080819,
- 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, 0x080819190808082b, 0x0808191908081919,
- 0x0808191908082b08, 0x0808191908190819, 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819,
- 0x0808191919081908, 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,
- 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, 0x0808192b1908082b,
- 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, 0x08082b0808081919, 0x08082b0808082b08,
- 0x08082b0808082b2b, 0x08082b0808190819, 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919,
- 0x08082b0819080819, 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,
- 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, 0x08082b1908190808,
- 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, 0x08082b2b08080808, 0x08082b2b082b0808,
- 0x08082b2b082b2b08, 0x08082b2b2b19192b, 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908,
- 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,
- 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, 0x081908081908082b,
- 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x08190808192b0808,
- 0x08190808192b2b2b, 0x081908082b080819, 0x081908082b081908, 0x081908082b190808, 0x0819081908080808,
- 0x081908190808082b, 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,
- 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, 0x081908192b080808,
- 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, 0x0819082b08081908, 0x0819082b0808192b,
- 0x0819082b08190808, 0x0819082b19080808, 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b,
- 0x0819190808081919, 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,
- 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, 0x08191908192b1908,
- 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, 0x0819191908190808, 0x0819191919080808,
- 0x0819192b08080808, 0x0819192b08191908, 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908,
- 0x08192b0808190808, 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,
- 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, 0x08192b2b2b2b2b19,
- 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, 0x082b080808082b2b,
- 0x082b080808190819, 0x082b080808191908, 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908,
- 0x082b080819190808, 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,
- 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, 0x082b082b08080808,
- 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,
- 0x082b190808190808, 0x082b1908082b2b19, 0x082b190819080808, 0x082b191908080808, 0x082b191919080819,
- 0x082b19191919082b, 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,
- 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, 0x082b2b0819191919,
- 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, 0x082b2b192b190808, 0x082b2b2b08082b08,
- 0x082b2b2b082b0808, 0x082b2b2b2b08082b, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,
- 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,
- 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808,
- 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, 0x1908080819190819,
- 0x1908080819191908, 0x19080808192b0808, 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908,
- 0x190808082b190808, 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,
- 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, 0x1908081919081908,
- 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, 0x190808192b2b082b, 0x1908082b08080819,
- 0x1908082b08081908, 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808,
- 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,
- 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, 0x1908190819081908,
- 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, 0x1908191908080819, 0x1908191908081908,
- 0x1908191908190808, 0x19081919082b1908, 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808,
- 0x1908192b08082b2b, 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,
- 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, 0x19082b08192b082b,
- 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, 0x19082b1919190808, 0x19082b19192b2b19,
- 0x19082b2b08081908, 0x1919080808080808, 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08,
- 0x1919080808190819, 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,
- 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, 0x1919081908081908,
- 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, 0x191908191908082b, 0x1919082b08080808,
- 0x1919082b19081908, 0x1919082b2b2b2b2b, 0x1919190808080819, 0x1919190808081908, 0x1919190808190808,
- 0x19191908082b0819, 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,
- 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, 0x1919192b082b0819,
- 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, 0x19192b0808191908, 0x19192b0819080819,
- 0x19192b0819190808, 0x19192b082b192b19, 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b,
- 0x19192b2b2b081919, 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,
- 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, 0x192b081908080808,
- 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, 0x192b190808080808, 0x192b19080819192b,
- 0x192b191908190808, 0x192b191919080808, 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819,
- 0x192b2b08192b2b2b, 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,
- 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,
- 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, 0x2b08080819080819, 0x2b08080819081908,
- 0x2b08080819190808, 0x2b0808082b080808, 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b,
- 0x2b08081908080819, 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,
- 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, 0x2b08082b2b080808,
- 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, 0x2b08190808080819, 0x2b08190808081908,
- 0x2b08190808190808, 0x2b0819080819082b, 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808,
- 0x2b0819082b082b19, 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,
- 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, 0x2b082b0819192b2b,
- 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, 0x2b082b190808192b, 0x2b082b2b082b082b,
- 0x2b082b2b2b080808, 0x2b082b2b2b082b08, 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819,
- 0x2b19080808081908, 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,
- 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, 0x2b19082b2b082b19,
- 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, 0x2b19190819190808, 0x2b19190819192b08,
- 0x2b191919082b2b19, 0x2b1919192b190808, 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819,
- 0x2b192b082b2b192b, 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,
- 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, 0x2b2b0808082b2b2b,
- 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, 0x2b2b08192b2b192b, 0x2b2b082b08080808,
- 0x2b2b082b0808082b, 0x2b2b082b08082b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808,
- 0x2b2b190819080808, 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,
- 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, 0x2b2b2b082b2b2b08,
- 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b,
- 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,
-};
-
-static const __device__ uint64_t iq2s_grid[1024] = {
- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b,
- 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, 0x0808080808192b19, 0x08080808082b0808,
- 0x08080808082b082b, 0x08080808082b1919, 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908,
- 0x080808081908192b, 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, 0x08080808192b2b19,
- 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,
- 0x080808082b191908, 0x080808082b2b0808, 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819,
- 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,
- 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808,
- 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,
- 0x080808191919192b, 0x0808081919192b19, 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08,
- 0x080808192b080819, 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,
- 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,
- 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, 0x0808082b082b2b2b,
- 0x0808082b19080819, 0x0808082b19081908, 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808,
- 0x0808082b19191919, 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,
- 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, 0x0808190808082b19,
- 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819,
- 0x08081908082b1908, 0x08081908082b192b, 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b,
- 0x0808190819081919, 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,
- 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, 0x08081908192b1919,
- 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, 0x080819082b082b19, 0x080819082b190808,
- 0x080819082b191919, 0x080819082b192b08, 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808,
- 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,
- 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, 0x08081919082b1919,
- 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, 0x080819191908192b, 0x0808191919082b19,
- 0x0808191919190808, 0x080819191919082b, 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819,
- 0x08081919192b1908, 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,
- 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, 0x0808192b08081908,
- 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b08191919, 0x0808192b19080808,
- 0x0808192b19081919, 0x0808192b19082b08, 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808,
- 0x0808192b2b080819, 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,
- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, 0x08082b080819192b,
- 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b08082b2b2b, 0x08082b0819080819,
- 0x08082b0819081908, 0x08082b081908192b, 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b,
- 0x08082b0819191919, 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,
- 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,
- 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, 0x08082b1908192b08, 0x08082b19082b0819,
- 0x08082b1919080808, 0x08082b1919081919, 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908,
- 0x08082b19192b0808, 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,
- 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, 0x08082b2b19190808,
- 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, 0x0819080808082b19,
- 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819,
- 0x08190808082b1908, 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,
- 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, 0x0819080819192b19,
- 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, 0x08190808192b2b08, 0x081908082b080819,
- 0x081908082b081908, 0x081908082b08192b, 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08,
- 0x081908082b2b0819, 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,
- 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, 0x081908190819192b,
- 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, 0x08190819082b1919, 0x08190819082b2b08,
- 0x0819081919080819, 0x0819081919081908, 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808,
- 0x081908191919082b, 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,
- 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, 0x081908192b190819,
- 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, 0x0819082b08082b19, 0x0819082b08190808,
- 0x0819082b08191919, 0x0819082b082b0819, 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919,
- 0x0819082b19190819, 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,
- 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, 0x0819190808190819,
- 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, 0x08191908082b0808, 0x08191908082b1919,
- 0x08191908082b2b08, 0x0819190819080819, 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19,
- 0x0819190819190808, 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,
- 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, 0x081919082b082b08,
- 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, 0x0819191908080819, 0x0819191908081908,
- 0x081919190808192b, 0x0819191908082b19, 0x0819191908190808, 0x081919190819082b, 0x0819191908191919,
- 0x0819191908192b08, 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,
- 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, 0x08191919192b0808,
- 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, 0x0819192b08080808, 0x0819192b08081919,
- 0x0819192b08082b08, 0x0819192b08190819, 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819,
- 0x0819192b19081908, 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,
- 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, 0x08192b0808191919,
- 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, 0x08192b081908082b, 0x08192b0819081919,
- 0x08192b0819082b08, 0x08192b0819190819, 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819,
- 0x08192b082b081908, 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,
- 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, 0x08192b1919081908,
- 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, 0x08192b2b08081908, 0x08192b2b08190808,
- 0x08192b2b19080808, 0x08192b2b1919192b, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,
- 0x082b080808082b08, 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,
- 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, 0x082b080819081908,
- 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, 0x082b0808192b1908, 0x082b08082b080808,
- 0x082b08082b082b2b, 0x082b08082b191908, 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908,
- 0x082b081908190808, 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,
- 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, 0x082b0819192b0808,
- 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, 0x082b082b08080808, 0x082b082b08082b2b,
- 0x082b082b082b082b, 0x082b082b082b2b08, 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808,
- 0x082b082b2b082b08, 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,
- 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, 0x082b190808192b08,
- 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, 0x082b19081908082b, 0x082b190819081919,
- 0x082b190819082b08, 0x082b190819190819, 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819,
- 0x082b19082b081908, 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,
- 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, 0x082b191919081908,
- 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, 0x082b192b08080819, 0x082b192b08081908,
- 0x082b192b08190808, 0x082b192b19080808, 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919,
- 0x082b2b0808190819, 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,
- 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, 0x082b2b1908190808,
- 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, 0x082b2b2b192b1908, 0x082b2b2b2b082b08,
- 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19,
- 0x1908080808190808, 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,
- 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, 0x190808081908082b,
- 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908,
- 0x190808081919192b, 0x1908080819192b19, 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919,
- 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,
- 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, 0x1908081908081919,
- 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, 0x190808190819192b, 0x1908081908192b19,
- 0x19080819082b0808, 0x19080819082b082b, 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908,
- 0x190808191908192b, 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,
- 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, 0x190808192b08082b,
- 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, 0x190808192b191908, 0x190808192b2b0808,
- 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919,
- 0x1908082b08192b08, 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,
- 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, 0x1908082b2b081908,
- 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808082b2b,
- 0x1908190808190819, 0x1908190808191908, 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808,
- 0x19081908082b082b, 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,
- 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, 0x1908190819191919,
- 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, 0x190819082b080808, 0x190819082b08082b,
- 0x190819082b081919, 0x190819082b082b08, 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808,
- 0x1908191908080819, 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,
- 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, 0x19081919082b1908,
- 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, 0x1908191919082b08, 0x1908191919190819,
- 0x1908191919191908, 0x19081919192b0808, 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908,
- 0x190819192b190808, 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,
- 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, 0x1908192b19081908,
- 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, 0x19082b0808080819, 0x19082b0808081908,
- 0x19082b0808082b19, 0x19082b0808190808, 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08,
- 0x19082b08082b0819, 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,
- 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, 0x19082b082b081908,
- 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, 0x19082b1908081919, 0x19082b1908082b08,
- 0x19082b1908190819, 0x19082b1908191908, 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908,
- 0x19082b1919190808, 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,
- 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, 0x1919080808081919,
- 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, 0x191908080819192b, 0x1919080808192b19,
- 0x19190808082b0808, 0x19190808082b082b, 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819,
- 0x1919080819081908, 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,
- 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, 0x191908082b080808,
- 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, 0x191908082b190819, 0x191908082b191908,
- 0x1919081908080819, 0x1919081908081908, 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808,
- 0x191908190819082b, 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,
- 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, 0x1919081919190819,
- 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, 0x191908192b081908, 0x191908192b190808,
- 0x1919082b08080808, 0x1919082b08081919, 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908,
- 0x1919082b082b0808, 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,
- 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, 0x1919190808082b19,
- 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, 0x1919190808192b08, 0x19191908082b0819,
- 0x19191908082b1908, 0x1919190819080808, 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08,
- 0x1919190819190819, 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,
- 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, 0x1919191908082b08,
- 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, 0x1919191919080819, 0x1919191919081908,
- 0x1919191919190808, 0x191919192b080808, 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808,
- 0x1919192b082b192b, 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,
- 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, 0x19192b0819080819,
- 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, 0x19192b082b080808, 0x19192b1908080819,
- 0x19192b1908081908, 0x19192b1908190808, 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19,
- 0x19192b2b2b081919, 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,
- 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, 0x192b0808082b0819,
- 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, 0x192b080819082b08, 0x192b080819190819,
- 0x192b080819191908, 0x192b0808192b0808, 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808,
- 0x192b08190808082b, 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,
- 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, 0x192b08192b080808,
- 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, 0x192b082b19080808, 0x192b082b1919192b,
- 0x192b082b2b2b0819, 0x192b190808080808, 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819,
- 0x192b190808191908, 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,
- 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, 0x192b191919080808,
- 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, 0x192b192b08080808, 0x192b192b2b191908,
- 0x192b2b0808080819, 0x192b2b0808081908, 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08,
- 0x192b2b1908080808, 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,
- 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, 0x2b08080808191908,
- 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, 0x2b08080819080819, 0x2b08080819081908,
- 0x2b08080819190808, 0x2b0808081919082b, 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819,
- 0x2b0808082b080808, 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,
- 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, 0x2b08081908191919,
- 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, 0x2b08081919080808, 0x2b0808191908082b,
- 0x2b08081919081919, 0x2b08081919082b08, 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819,
- 0x2b0808192b081908, 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,
- 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, 0x2b08082b19081908,
- 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, 0x2b0819080808192b, 0x2b08190808082b19,
- 0x2b08190808190808, 0x2b0819080819082b, 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819,
- 0x2b08190819080808, 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,
- 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, 0x2b0819082b190808,
- 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, 0x2b08191908082b08, 0x2b08191908190819,
- 0x2b08191908191908, 0x2b081919082b0808, 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808,
- 0x2b0819192b080808, 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,
- 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, 0x2b082b0808190819,
- 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, 0x2b082b0819190808, 0x2b082b082b2b082b,
- 0x2b082b1908080819, 0x2b082b1908081908, 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b,
- 0x2b082b2b19192b08, 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,
- 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, 0x2b19080808191919,
- 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908081908082b, 0x2b19080819081919,
- 0x2b19080819082b08, 0x2b19080819190819, 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819,
- 0x2b1908082b081908, 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,
- 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, 0x2b19081919192b2b,
- 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, 0x2b19082b19080808, 0x2b19082b2b2b192b,
- 0x2b19190808080808, 0x2b1919080808082b, 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819,
- 0x2b19190808191908, 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,
- 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, 0x2b19191908190808,
- 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, 0x2b19192b08080808, 0x2b19192b1908192b,
- 0x2b19192b192b1908, 0x2b192b0808080819, 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b,
- 0x2b192b0819080808, 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,
- 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, 0x2b2b080808191908,
- 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, 0x2b2b080819081908, 0x2b2b080819190808,
- 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b,
- 0x2b2b082b08082b2b, 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,
- 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, 0x2b2b190808081908,
- 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, 0x2b2b19082b2b1908, 0x2b2b191908080808,
- 0x2b2b191908192b19, 0x2b2b192b19190819, 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b,
- 0x2b2b2b1919191908, 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,
- 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,
-};
-
-static const __device__ uint32_t iq3xxs_grid[256] = {
- 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, 0x04041c0c,
- 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, 0x040c140c, 0x040c142c,
- 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, 0x04140414, 0x04140424, 0x04140c0c,
- 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c,
- 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e,
- 0x04243e1c, 0x04243e2c, 0x042c040c, 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04,
- 0x043e0c24, 0x043e0c34, 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c,
- 0x0c04141c, 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,
- 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, 0x0c143e14,
- 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, 0x0c24042c, 0x0c242c04,
- 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, 0x0c3e2404, 0x14040404, 0x14040414,
- 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c,
- 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404,
- 0x14140414, 0x14140c0c, 0x14140c3e, 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c,
- 0x141c0c04, 0x141c0c24, 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c,
- 0x142c3e24, 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,
- 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, 0x1c0c2424,
- 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, 0x1c1c0c0c, 0x1c1c1c1c,
- 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, 0x1c2c2c2c, 0x1c340c24, 0x1c341c34,
- 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e,
- 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424,
- 0x24242c0c, 0x24243424, 0x242c142c, 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04,
- 0x2c040c14, 0x2c04240c, 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14,
- 0x2c143e14, 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,
- 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, 0x340c340c,
- 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, 0x34341c1c, 0x343e041c,
- 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, 0x3e042c14, 0x3e0c1434, 0x3e0c2404,
- 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c,
- 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,
-};
-
-static const __device__ uint32_t iq3xs_grid[512] = {
- 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, 0x04040c24,
- 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, 0x0404242c, 0x0404243e,
- 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, 0x04043e3e, 0x040c0404, 0x040c040c,
- 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c,
- 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e,
- 0x04140c04, 0x04140c1c, 0x04140c34, 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c,
- 0x0414243e, 0x04142c0c, 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404,
- 0x041c1414, 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c,
- 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, 0x0424241c,
- 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, 0x042c1c1c, 0x042c240c,
- 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, 0x04340c1c, 0x04341c0c, 0x04342c14,
- 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04,
- 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c,
- 0x0c040c3e, 0x0c041404, 0x0c041414, 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c,
- 0x0c043e14, 0x0c0c0404, 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04,
- 0x0c0c1c1c, 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c,
- 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, 0x0c143e14,
- 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, 0x0c1c1c04, 0x0c1c1c24,
- 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, 0x0c240c24, 0x0c241c0c, 0x0c241c1c,
- 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04,
- 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c,
- 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c,
- 0x1404041c, 0x1404042c, 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c,
- 0x1404143e, 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e,
- 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, 0x140c1414,
- 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, 0x1414043e, 0x1414140c,
- 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, 0x14143e0c, 0x14143e24, 0x141c0404,
- 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04,
- 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14,
- 0x14243e2c, 0x142c0424, 0x142c0c0c, 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404,
- 0x14340414, 0x1434043e, 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04,
- 0x143e241c, 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c,
- 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, 0x1c0c040c,
- 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, 0x1c0c3e14, 0x1c0c3e34,
- 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24,
- 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c,
- 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c,
- 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c,
- 0x1c3e040c, 0x1c3e041c, 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404,
- 0x24041424, 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c,
- 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, 0x2414041c,
- 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, 0x24143e04, 0x241c0424,
- 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, 0x24240404, 0x24240414, 0x24241424,
- 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24,
- 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24,
- 0x2c041414, 0x2c042404, 0x2c042424, 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c,
- 0x2c0c042c, 0x2c0c0c14, 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04,
- 0x2c141c34, 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c,
- 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, 0x2c2c2c0c,
- 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, 0x34040c2c, 0x34041c0c,
- 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, 0x34140c14, 0x34141c24, 0x34142414,
- 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c,
- 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c,
- 0x3e040404, 0x3e040424, 0x3e04043e, 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414,
- 0x3e0c0414, 0x3e0c0c0c, 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34,
- 0x3e14140c, 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c,
- 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404,
-};
-
-#define IQ1S_DELTA 0.125f
-#define IQ1M_DELTA 0.125f
-static const __device__ uint64_t iq1s_grid_gpu[2048] = {
- 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000, 0x00020002,
- 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101, 0x02000000, 0x02000002,
- 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200, 0x02020202, 0x00000110, 0x00000111,
- 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212, 0x00020111, 0x01000011, 0x01000112, 0x01000211,
- 0x01010012, 0x01010111, 0x01010212, 0x01020011, 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011,
- 0x02010110, 0x02010112, 0x02020111, 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020,
- 0x00020022, 0x00020220, 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020,
- 0x02000022, 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,
- 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101, 0x01011202,
- 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110, 0x00001111, 0x00001112,
- 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111, 0x01001212, 0x01011010, 0x01011011,
- 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010, 0x01021012, 0x01021111, 0x01021210, 0x01021212,
- 0x02001011, 0x02011011, 0x02011111, 0x02011210, 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112,
- 0x02021211, 0x00011120, 0x00011221, 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220,
- 0x01021020, 0x01021021, 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000,
- 0x00002002, 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,
- 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101, 0x02022000,
- 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211, 0x00022110, 0x00022111,
- 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110, 0x01022211, 0x02012011, 0x02012110,
- 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022, 0x00002220, 0x00002222, 0x00012121, 0x00022020,
- 0x00022022, 0x00022220, 0x00022222, 0x01002121, 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020,
- 0x02002022, 0x02002121, 0x02002220, 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222,
- 0x00110000, 0x00110001, 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000,
- 0x01110101, 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,
- 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012, 0x00110111,
- 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010, 0x01110011, 0x01110012,
- 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111, 0x02100110, 0x02110012, 0x02110111,
- 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122, 0x00120121, 0x01100020, 0x01100122, 0x01100221,
- 0x01110022, 0x01110121, 0x01110220, 0x01110222, 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120,
- 0x02110122, 0x02120121, 0x00101001, 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201,
- 0x00121001, 0x00121102, 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100,
- 0x01111101, 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,
- 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101, 0x02121201,
- 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112, 0x00111211, 0x00121010,
- 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110, 0x01101111, 0x01101112, 0x01111011,
- 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211, 0x01111212, 0x01121011, 0x01121110, 0x01121111,
- 0x01121112, 0x01121211, 0x02101010, 0x02101012, 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010,
- 0x02111011, 0x02111110, 0x02111111, 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111,
- 0x00101021, 0x00101120, 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021,
- 0x00121122, 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,
- 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221, 0x01121222,
- 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001, 0x00112102, 0x00122101,
- 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101, 0x01112200, 0x01112202, 0x01122000,
- 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101, 0x02112001, 0x02112100, 0x02122101, 0x00112010,
- 0x00112012, 0x00112111, 0x00112212, 0x00122011, 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210,
- 0x01112011, 0x01112110, 0x01112111, 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212,
- 0x02102211, 0x02112011, 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221,
- 0x00112122, 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,
- 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222, 0x00200000,
- 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101, 0x00220200, 0x00220202,
- 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000, 0x02200002, 0x02200200, 0x02200202,
- 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200, 0x02220202, 0x00200111, 0x00210011, 0x00210110,
- 0x00210211, 0x00220111, 0x01200012, 0x01200110, 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011,
- 0x01220110, 0x01220111, 0x01220112, 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021,
- 0x00200220, 0x00200222, 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121,
- 0x01210021, 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,
- 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201, 0x00221101,
- 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200, 0x01211202, 0x01221102,
- 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101, 0x00201211, 0x00211111, 0x00221011,
- 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011, 0x01211110, 0x01211111, 0x01211211, 0x01221012,
- 0x01221111, 0x01221210, 0x02201211, 0x02211010, 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011,
- 0x02221110, 0x02221112, 0x02221211, 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021,
- 0x01201221, 0x01211121, 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222,
- 0x00202000, 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,
- 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202, 0x02222000,
- 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211, 0x00222111, 0x01202112,
- 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112, 0x01222211, 0x02202111, 0x02212010,
- 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020, 0x00202022, 0x00202220, 0x00202222, 0x00222020,
- 0x00222022, 0x00222220, 0x00222222, 0x01202121, 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020,
- 0x02202022, 0x02202220, 0x02202222, 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101,
- 0x10010001, 0x10010102, 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001,
- 0x11020100, 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,
- 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011, 0x10020112,
- 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111, 0x11010112, 0x11010211,
- 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110, 0x12000112, 0x12010010, 0x12010012,
- 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121, 0x10010021, 0x10010120, 0x10010122, 0x10020121,
- 0x11000021, 0x11010022, 0x11010121, 0x11010222, 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121,
- 0x10001001, 0x10011101, 0x10011201, 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100,
- 0x11011101, 0x11011102, 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102,
- 0x12001201, 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,
- 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010, 0x10021111,
- 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010, 0x11011011, 0x11011110,
- 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110, 0x11021111, 0x11021112, 0x11021211,
- 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011, 0x12011110, 0x12011111, 0x12011112, 0x12011211,
- 0x12011212, 0x12021111, 0x12021210, 0x12021212, 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121,
- 0x10011220, 0x10011222, 0x10021021, 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220,
- 0x11011020, 0x11011021, 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220,
- 0x12001021, 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,
- 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101, 0x11012200,
- 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100, 0x12012102, 0x12012201,
- 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010, 0x10012110, 0x10012111, 0x10012210,
- 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111, 0x11002212, 0x11012011, 0x11012012, 0x11012110,
- 0x11012111, 0x11012112, 0x11012211, 0x11022010, 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112,
- 0x12002211, 0x12012012, 0x12012111, 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211,
- 0x10012122, 0x11002120, 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221,
- 0x12012120, 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,
- 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001, 0x11110100,
- 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201, 0x12110101, 0x12110200,
- 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210, 0x10100211, 0x10100212, 0x10110011,
- 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211, 0x10120010, 0x10120111, 0x10120112, 0x10120210,
- 0x10120212, 0x11100011, 0x11100110, 0x11100111, 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012,
- 0x11110110, 0x11110111, 0x11110112, 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111,
- 0x11120112, 0x11120211, 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211,
- 0x12120010, 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,
- 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122, 0x11110221,
- 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221, 0x12110222, 0x12120120,
- 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102, 0x10111200, 0x10111201, 0x10121001,
- 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100, 0x11101101, 0x11101102, 0x11101201, 0x11101202,
- 0x11111000, 0x11111001, 0x11111100, 0x11111101, 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001,
- 0x11121002, 0x11121100, 0x11121101, 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001,
- 0x12111100, 0x12111101, 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011,
- 0x10101012, 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,
- 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112, 0x10121211,
- 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210, 0x11101211, 0x11111010,
- 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210, 0x11111211, 0x11111212, 0x11121010,
- 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210, 0x11121211, 0x11121212, 0x12101011, 0x12101110,
- 0x12101111, 0x12101211, 0x12101212, 0x12111010, 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210,
- 0x12111211, 0x12121011, 0x12121110, 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022,
- 0x10101120, 0x10101122, 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221,
- 0x10121020, 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,
- 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022, 0x11111120,
- 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120, 0x11121121, 0x11121221,
- 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222, 0x12111021, 0x12111121, 0x12111222,
- 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221, 0x10102100, 0x10102101, 0x10102102, 0x10102201,
- 0x10112000, 0x10112101, 0x10112200, 0x10122001, 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001,
- 0x11112100, 0x11112101, 0x11112102, 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101,
- 0x12102002, 0x12102201, 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011,
- 0x10102012, 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,
- 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012, 0x11112110,
- 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110, 0x11122111, 0x11122112,
- 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110, 0x12112111, 0x12112112, 0x12112210,
- 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121, 0x10112222, 0x10122020, 0x10122121, 0x10122122,
- 0x10122221, 0x11102121, 0x11102220, 0x11102221, 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221,
- 0x11122022, 0x11122121, 0x11122220, 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122,
- 0x12112220, 0x12112222, 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100,
- 0x11210000, 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,
- 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012, 0x10210111,
- 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011, 0x11210111, 0x11210112,
- 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212, 0x12210012, 0x12210111, 0x12220011,
- 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221, 0x11200020, 0x11200021, 0x11200122, 0x11210121,
- 0x11210122, 0x11210220, 0x11220020, 0x12200121, 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002,
- 0x10211101, 0x10211102, 0x10211202, 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101,
- 0x11201200, 0x11201202, 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000,
- 0x11221002, 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,
- 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210, 0x10201212,
- 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112, 0x11201211, 0x11211010,
- 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011, 0x11221110, 0x11221111, 0x11221112,
- 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011, 0x12211111, 0x12211112, 0x12211211, 0x12211212,
- 0x12221012, 0x12221111, 0x12221112, 0x12221210, 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122,
- 0x10221220, 0x10221221, 0x11201020, 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121,
- 0x11211122, 0x11211220, 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121,
- 0x12201222, 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,
- 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001, 0x11222201,
- 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010, 0x10212111, 0x10222011,
- 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111, 0x11202112, 0x11202210, 0x11212011,
- 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010, 0x11222111, 0x11222212, 0x12202012, 0x12202110,
- 0x12202212, 0x12212111, 0x12222011, 0x12222110, 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220,
- 0x11202021, 0x11202120, 0x11202221, 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121,
- 0x11222221, 0x12202122, 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200,
- 0x20000202, 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,
- 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101, 0x22020000,
- 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112, 0x20010211, 0x20020111,
- 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111, 0x21010112, 0x21010210, 0x21010211,
- 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211, 0x22010110, 0x22010112, 0x22010211, 0x22020111,
- 0x20000020, 0x20000022, 0x20000220, 0x20000222, 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222,
- 0x21010021, 0x21010120, 0x21010221, 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121,
- 0x22020020, 0x22020022, 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001,
- 0x21011101, 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,
- 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111, 0x21001210,
- 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111, 0x21021112, 0x21021210,
- 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010, 0x22011012, 0x22011111, 0x22011210,
- 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121, 0x21001021, 0x21001120, 0x21001221, 0x21001222,
- 0x21011020, 0x21011121, 0x21011221, 0x21011222, 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021,
- 0x22011222, 0x22021120, 0x20002000, 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002,
- 0x20022200, 0x20022202, 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201,
- 0x22002000, 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,
- 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110, 0x21002112,
- 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110, 0x22002111, 0x22012112,
- 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222, 0x20012121, 0x20022020, 0x20022022,
- 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120, 0x21012122, 0x22002020, 0x22002022, 0x22002220,
- 0x22002222, 0x22012121, 0x22022020, 0x22022022, 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102,
- 0x20110200, 0x20110201, 0x20120101, 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202,
- 0x21120201, 0x21120202, 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011,
- 0x20100110, 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,
- 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111, 0x21110112,
- 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111, 0x22110210, 0x22120011,
- 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120, 0x20110221, 0x20120121, 0x21100120,
- 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121, 0x21110220, 0x21120122, 0x21120221, 0x22100121,
- 0x22110120, 0x22110122, 0x22120221, 0x20101001, 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200,
- 0x20121102, 0x21101000, 0x21101202, 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201,
- 0x21121000, 0x21121001, 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101,
- 0x22111200, 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,
- 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212, 0x21101011,
- 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012, 0x21111110, 0x21111111,
- 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110, 0x21121111, 0x21121112, 0x21121211,
- 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012, 0x22111110, 0x22111111, 0x22111112, 0x22111211,
- 0x22111212, 0x22121010, 0x22121012, 0x22121111, 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020,
- 0x20111121, 0x20111221, 0x20121020, 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021,
- 0x21111022, 0x21111121, 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221,
- 0x22101222, 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,
- 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102, 0x21112202,
- 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101, 0x20102110, 0x20102112,
- 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212, 0x20122010, 0x20122011, 0x20122110,
- 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210, 0x21102212, 0x21112011, 0x21112110, 0x21112111,
- 0x21112112, 0x21112211, 0x21122012, 0x21122111, 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010,
- 0x22112012, 0x22112111, 0x22112212, 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120,
- 0x21102122, 0x21102221, 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120,
- 0x22112121, 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,
- 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000, 0x22200002,
- 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202, 0x20200111, 0x20200211,
- 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112, 0x21200211, 0x21210011, 0x21210111,
- 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111, 0x22210010, 0x22210012, 0x22210112, 0x22210211,
- 0x20200022, 0x20200220, 0x20200222, 0x20210020, 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121,
- 0x21210021, 0x21210122, 0x21210221, 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121,
- 0x22220020, 0x22220022, 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000,
- 0x21211100, 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,
- 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112, 0x20221211,
- 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211, 0x21221111, 0x21221212,
- 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012, 0x22211111, 0x22211210, 0x20201121,
- 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121, 0x21201120, 0x21201122, 0x21201222, 0x21211022,
- 0x21211121, 0x21211122, 0x21211220, 0x21221020, 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122,
- 0x22211221, 0x22221021, 0x22221120, 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000,
- 0x20222002, 0x20222200, 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002,
- 0x22202200, 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,
- 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011, 0x21222112,
- 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222, 0x20222020, 0x20222022,
- 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020, 0x22202022, 0x22202220, 0x22202222,
- 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,
-};
-
-static const __device__ uint8_t ksigns_iq2xs[128] = {
- 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, 144, 17, 18, 147, 20, 149,
- 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43,
- 172, 45, 46, 175, 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, 192, 65,
- 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, 80, 209, 210, 83, 212, 85, 86, 215,
- 216, 89, 90, 219, 92, 221, 222, 95, 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237,
- 238, 111, 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
-};
-
-static const __device__ uint64_t ksigns64[128] = {
- 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff, 0xff00000000ff0000,
- 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff, 0xff000000ff000000, 0x00000000ff0000ff,
- 0x00000000ff00ff00, 0xff000000ff00ffff, 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00,
- 0x00000000ffffffff, 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,
- 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff, 0x000000ffff000000,
- 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff, 0xff0000ffffff0000, 0x000000ffffff00ff,
- 0x000000ffffffff00, 0xff0000ffffffffff, 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00,
- 0xff00ff000000ffff, 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,
- 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff, 0xff00ff00ffff0000,
- 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff, 0x0000ffff00000000, 0xff00ffff000000ff,
- 0xff00ffff0000ff00, 0x0000ffff0000ffff, 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00,
- 0xff00ffff00ffffff, 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,
- 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff, 0xffff000000000000,
- 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff, 0x00ff000000ff0000, 0xffff000000ff00ff,
- 0xffff000000ffff00, 0x00ff000000ffffff, 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00,
- 0x00ff0000ff00ffff, 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,
- 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff, 0xffff00ff00ff0000,
- 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff, 0xffff00ffff000000, 0x00ff00ffff0000ff,
- 0x00ff00ffff00ff00, 0xffff00ffff00ffff, 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00,
- 0x00ff00ffffffffff, 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,
- 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff, 0xffffff00ff000000,
- 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff, 0x00ffff00ffff0000, 0xffffff00ffff00ff,
- 0xffffff00ffffff00, 0x00ffff00ffffffff, 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00,
- 0xffffffff0000ffff, 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,
- 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff, 0xffffffffffff0000,
- 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,
-};
-
-static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};
-static const __device__ int8_t kvalues_iq4nl[16] = {
- -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
-
-typedef half dfloat; // dequantize float
-typedef half2 dfloat2;
-typedef void (*dequantize_kernel_t)(const void* vx, const int ib, const int iqs, dfloat2& v);
-template <typename dst_t>
-using to_cuda_ggml_t = void (*)(const void* __restrict__ x, dst_t* __restrict__ y, int k, cudaStream_t stream);
-typedef float (*vec_dot_q_cuda_t)(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs);
-typedef void (*allocate_tiles_cuda_t)(int** x_ql, half2** x_dm, int** x_qh, int** x_sc);
-typedef void (*load_tiles_cuda_t)(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row);
-typedef float (*vec_dot_q_mul_mat_cuda_t)(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ms,
- const int& i,
- const int& j,
- const int& k);
-
-// Utility function
-
-template <typename dst_t>
-static __device__ __forceinline__ dst_t convert_from_half(half val) {
- return val;
-}
-
-template <>
-__device__ __forceinline__ c10::BFloat16 convert_from_half<c10::BFloat16>(half val) {
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA)
- return __float2bfloat16(__half2float(val));
-#else
- return __half2float(val);
-#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA)
-}
-
-template <>
-__device__ __forceinline__ float convert_from_half<float>(half val) {
- return __half2float(val);
-}
-
-#if defined(USE_ROCM)
-
-#ifndef __has_builtin
-#define __has_builtin(x) 0
-#endif
-
-typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));
-static __device__ __forceinline__ int __vsubss4(const int a, const int b) {
- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
-#if __has_builtin(__builtin_elementwise_sub_sat)
- const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);
- return reinterpret_cast<const int&>(c);
-#else
- int8x4_t c;
- int16_t tmp;
-#pragma unroll
- for (int i = 0; i < 4; i++) {
- tmp = va[i] - vb[i];
- if (tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();
- if (tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();
- c[i] = tmp;
- }
- return reinterpret_cast<int&>(c);
-#endif // __has_builtin(__builtin_elementwise_sub_sat)
-}
-
-static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {
-#if __has_builtin(__builtin_amdgcn_sdot4)
- c = __builtin_amdgcn_sdot4(a, b, c, false);
-#else
- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);
- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);
- c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];
-#endif
- return c;
-}
-
-static __device__ __forceinline__ uint32_t __vcmpeq4(const uint32_t a, const uint32_t b) {
- uint32_t neq = a ^ b;
- return !(neq & 0xff000000) * 0xff000000 | !(neq & 0x00ff0000) * 0x00ff0000 | !(neq & 0x0000ff00) * 0x0000ff00 |
- !(neq & 0x000000ff) * 0x000000ff;
-}
-
-static __device__ __forceinline__ uint32_t __vsub4(const uint32_t a, const uint32_t b) {
- return (static_cast<uint8_t>(((a & 0xff000000) >> 24) - ((b & 0xff000000) >> 24)) << 24) +
- (static_cast<uint8_t>(((a & 0x00ff0000) >> 16) - ((b & 0x00ff0000) >> 16)) << 16) +
- (static_cast<uint8_t>(((a & 0x0000ff00) >> 8) - ((b & 0x0000ff00) >> 8)) << 8) +
- (static_cast<uint8_t>(((a & 0x000000ff) >> 0) - ((b & 0x000000ff) >> 0)) << 0);
-}
-#endif // defined(USE_ROCM)
diff --git a/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu b/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu
deleted file mode 100644
index d88960d5..00000000
--- a/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu
+++ /dev/null
@@ -1,848 +0,0 @@
-// Adatped from
-// https://github.com/vllm-project/vllm/blob/755ed7b05be4743237d3339c4ff8c22bcaae04f4/csrc/quantization/gguf/gguf_kernel.cu
-#include <c10/cuda/CUDAGuard.h>
-#include <cuda_fp16.h>
-#include <cuda_runtime.h>
-#include <torch/all.h>
-
-// dont use clang-format here, it breaks the include order
-// clang-format off
-#include "dispatch.h"
-
-#include "ggml-common.h"
-#include "vecdotq.cuh"
-#include "dequantize.cuh"
-#include "mmvq.cuh"
-#include "mmq.cuh"
-#include "moe.cuh"
-#include "moe_vec.cuh"
-// clang-format off
-
-// Q8 gemv
-template <typename scalar_t>
-static __global__ void
-quantize_q8_1(const scalar_t* __restrict__ x, void* __restrict__ vy, const int kx, const int kx_padded) {
- const auto ix = blockDim.x * blockIdx.x + threadIdx.x;
- if (ix >= kx_padded) {
- return;
- }
- const auto iy = blockDim.y * blockIdx.y + threadIdx.y;
- const int i_padded = iy * kx_padded + ix;
-
- block_q8_1* y = (block_q8_1*)vy;
-
- const int ib = i_padded / QK8_1; // block index
- const int iqs = i_padded % QK8_1; // quant index
-
- const float xi = ix < kx ? static_cast<float>(x[iy * kx + ix]) : 0.0f;
- float amax = fabsf(xi);
- float sum = xi;
-
-#pragma unroll
- for (int mask = 16; mask > 0; mask >>= 1) {
- amax = fmaxf(amax, SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), amax, mask, 32));
- sum += SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), sum, mask, 32);
- }
-
- const float d = amax / 127;
- const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
-
- y[ib].qs[iqs] = q;
-
- if (iqs > 0) {
- return;
- }
-
- y[ib].ds.x = __float2half(d);
- y[ib].ds.y = __float2half(sum);
-}
-
-template <typename scalar_t>
-static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx, const int ky, cudaStream_t stream) {
- const int64_t kx_padded = (kx + 512 - 1) / 512 * 512;
- const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
- constexpr int MAX_BLOCK_SIZE = 65535;
- for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) {
- const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off;
- const dim3 num_blocks(block_num_x, num_blocks_y, 1);
- const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);
- quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(
- &x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded);
- }
-}
-
-torch::Tensor ggml_dequantize(
- torch::Tensor W, // quant weight
- int64_t type,
- int64_t m,
- int64_t n,
- std::optional<at::ScalarType> const& dtype) {
- const at::cuda::OptionalCUDAGuard device_guard(device_of(W));
- auto dtype_ = dtype.value_or(torch::kFloat16);
- auto options = torch::TensorOptions().dtype(dtype_).device(W.device());
- at::Tensor DW = torch::empty({m, n}, options);
- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
-
- DISPATCH_FLOAT_TYPES(DW.scalar_type(), "ggml_dequantize", [&] {
- auto to_cuda = ggml_get_to_cuda<scalar_t>(type);
- to_cuda((void*)W.data_ptr(), (scalar_t*)DW.data_ptr(), m * n, stream);
- });
-
- return DW;
-}
-
-torch::Tensor ggml_mul_mat_vec_a8(
- torch::Tensor W, // quant weight
- torch::Tensor X, // input
- int64_t type,
- int64_t row) {
- int col = X.sizes()[1];
- int vecs = X.sizes()[0];
- const int padded = (col + 512 - 1) / 512 * 512;
- const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
- auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device());
- at::Tensor Y = torch::empty({vecs, row}, options);
- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
- options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());
- at::Tensor quant_X = torch::empty({vecs, padded / 32 * 9}, options);
- DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_mul_mat_vec_a8", [&] {
- quantize_row_q8_1_cuda<scalar_t>((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, vecs, stream);
- switch (type) {
- case 2:
- mul_mat_vec_q4_0_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 3:
- mul_mat_vec_q4_1_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 6:
- mul_mat_vec_q5_0_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 7:
- mul_mat_vec_q5_1_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 8:
- mul_mat_vec_q8_0_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 10:
- mul_mat_vec_q2_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 11:
- mul_mat_vec_q3_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 12:
- mul_mat_vec_q4_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 13:
- mul_mat_vec_q5_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 14:
- mul_mat_vec_q6_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 16:
- mul_mat_vec_iq2_xxs_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 17:
- mul_mat_vec_iq2_xs_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 18:
- mul_mat_vec_iq3_xxs_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 19:
- mul_mat_vec_iq1_s_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 20:
- mul_mat_vec_iq4_nl_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 21:
- mul_mat_vec_iq3_s_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 22:
- mul_mat_vec_iq2_s_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 23:
- mul_mat_vec_iq4_xs_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- case 29:
- mul_mat_vec_iq1_m_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);
- break;
- }
- });
- return Y;
-}
-
-torch::Tensor ggml_mul_mat_a8(
- torch::Tensor W, // quant weight
- torch::Tensor X, // input
- int64_t type,
- int64_t row) {
- int col = X.sizes()[1];
- int padded = (col + 512 - 1) / 512 * 512;
- int batch = X.sizes()[0];
- const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
- auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device());
- at::Tensor Y = torch::empty({batch, row}, options);
- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
- options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());
- at::Tensor quant_X = torch::empty({batch, padded / 32 * 9}, options);
- DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_mul_mat_a8", [&] {
- quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, batch, stream);
-
- switch (type) {
- case 2:
- ggml_mul_mat_q4_0_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 3:
- ggml_mul_mat_q4_1_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 6:
- ggml_mul_mat_q5_0_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 7:
- ggml_mul_mat_q5_1_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 8:
- ggml_mul_mat_q8_0_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 10:
- ggml_mul_mat_q2_K_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 11:
- ggml_mul_mat_q3_K_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 12:
- ggml_mul_mat_q4_K_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 13:
- ggml_mul_mat_q5_K_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- case 14:
- ggml_mul_mat_q6_K_q8_1_cuda(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- col,
- row,
- batch,
- padded,
- row,
- stream);
- break;
- }
- });
- return Y;
-}
-
-torch::Tensor ggml_moe_a8(
- torch::Tensor X, // input
- torch::Tensor W, // expert weights
- torch::Tensor sorted_token_ids,
- torch::Tensor expert_ids,
- torch::Tensor num_tokens_post_padded,
- int64_t type,
- int64_t row,
- int64_t top_k,
- int64_t tokens) {
- int col = X.sizes()[1];
- int padded = (col + 512 - 1) / 512 * 512;
- const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
- auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device());
- at::Tensor Y = torch::empty({tokens * top_k, row}, options);
- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
- options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());
- at::Tensor quant_X = torch::empty({tokens, padded / 32 * 9}, options);
- DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_moe_a8", [&] {
- quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, tokens, stream);
- switch (type) {
- case 2:
- ggml_moe_q4_0_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 3:
- ggml_moe_q4_1_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 6:
- ggml_moe_q5_0_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 7:
- ggml_moe_q5_1_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 8:
- ggml_moe_q8_0_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 10:
- ggml_moe_q2_K_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 11:
- ggml_moe_q3_K_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 12:
- ggml_moe_q4_K_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 13:
- ggml_moe_q5_K_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- case 14:
- ggml_moe_q6_K_q8_1_cuda(
- (void*)quant_X.data_ptr(),
- (void*)W.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)sorted_token_ids.data_ptr(),
- (int*)expert_ids.data_ptr(),
- (int*)num_tokens_post_padded.data_ptr(),
- W.stride(0),
- col,
- row,
- tokens,
- padded,
- row,
- top_k,
- sorted_token_ids.sizes()[0],
- stream);
- break;
- }
- });
- return Y;
-}
-
-torch::Tensor ggml_moe_a8_vec(
- torch::Tensor X, // input
- torch::Tensor W, // expert weights
- torch::Tensor topk_ids,
- int64_t top_k,
- int64_t type,
- int64_t row,
- int64_t tokens) {
- int col = X.sizes()[1];
- const int padded = (col + 512 - 1) / 512 * 512;
- const at::cuda::OptionalCUDAGuard device_guard(device_of(X));
- auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device());
- at::Tensor Y = torch::zeros({tokens * top_k, row}, options);
- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();
- options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());
- at::Tensor quant_X = torch::empty({tokens, padded / 32 * 9}, options);
- DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_moe_vec_a8", [&] {
- quantize_row_q8_1_cuda<scalar_t>((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, tokens, stream);
- switch (type) {
- case 2:
- moe_vec_q4_0_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 3:
- moe_vec_q4_1_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 6:
- moe_vec_q5_0_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 7:
- moe_vec_q5_1_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 8:
- moe_vec_q8_0_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 10:
- moe_vec_q2_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 11:
- moe_vec_q3_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 12:
- moe_vec_q4_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 13:
- moe_vec_q5_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 14:
- moe_vec_q6_K_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 16:
- moe_vec_iq2_xxs_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 17:
- moe_vec_iq2_xs_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 18:
- moe_vec_iq3_xxs_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 19:
- moe_vec_iq1_s_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 20:
- moe_vec_iq4_nl_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 21:
- moe_vec_iq3_s_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 22:
- moe_vec_iq2_s_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 23:
- moe_vec_iq4_xs_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- case 29:
- moe_vec_iq1_m_q8_1_cuda<scalar_t>(
- (void*)W.data_ptr(),
- (void*)quant_X.data_ptr(),
- (scalar_t*)Y.data_ptr(),
- (int*)topk_ids.data_ptr(),
- top_k,
- tokens,
- col,
- row,
- quant_X.stride(0),
- stream);
- break;
- }
- });
- return Y;
-}
-
-int64_t ggml_moe_get_block_size(int64_t type) {
- switch (type) {
- case 2:
- return MOE_X_Q4_0;
- case 3:
- return MOE_X_Q4_1;
- case 6:
- return MOE_X_Q5_0;
- case 7:
- return MOE_X_Q5_1;
- case 8:
- return MOE_X_Q8_0;
- case 10:
- return MOE_X_Q2_K;
- case 11:
- return MOE_X_Q3_K;
- case 12:
- return MOE_X_Q4_K;
- case 13:
- return MOE_X_Q5_K;
- case 14:
- return MOE_X_Q6_K;
- }
- return 0;
-}
-
-// ---- FreeToken pybind bindings (donor registers these via TORCH_LIBRARY; we
-// expose them through torch.utils.cpp_extension.load's pybind module instead) ----
-#include <torch/extension.h>
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
- m.def("ggml_dequantize", &ggml_dequantize, "");
- m.def("ggml_mul_mat_vec_a8", &ggml_mul_mat_vec_a8, "");
- m.def("ggml_mul_mat_a8", &ggml_mul_mat_a8, "");
- m.def("ggml_moe_a8", &ggml_moe_a8, "");
- m.def("ggml_moe_a8_vec", &ggml_moe_a8_vec, "");
- m.def("ggml_moe_get_block_size", &ggml_moe_get_block_size, "");
-}
diff --git a/python/freetoken/kernel/csrc/gguf/mmq.cuh b/python/freetoken/kernel/csrc/gguf/mmq.cuh
deleted file mode 100644
index 9838c14f..00000000
--- a/python/freetoken/kernel/csrc/gguf/mmq.cuh
+++ /dev/null
@@ -1,881 +0,0 @@
-// copied from
-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/mmq.cuh
-// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
-template <
- typename scalar_t,
- int qk,
- int qr,
- int qi,
- bool need_sum,
- typename block_q_t,
- int mmq_x,
- int mmq_y,
- int nwarps,
- allocate_tiles_cuda_t allocate_tiles,
- load_tiles_cuda_t load_tiles,
- int vdr,
- vec_dot_q_mul_mat_cuda_t vec_dot>
-static __device__ __forceinline__ void mul_mat_q(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const block_q_t* x = (const block_q_t*)vx;
- const block_q8_1* y = (const block_q8_1*)vy;
-
- const int blocks_per_row_x = ncols_x / qk;
- const int blocks_per_col_y = nrows_y / QK8_1;
- const int blocks_per_warp = WARP_SIZE_GGUF / qi;
-
- const int& ncols_dst = ncols_y;
-
- const auto row_dst_0 = blockIdx.x * mmq_y;
- const int& row_x_0 = row_dst_0;
-
- const auto col_dst_0 = blockIdx.y * mmq_x;
- const int& col_y_0 = col_dst_0;
-
- int* tile_x_ql = nullptr;
- half2* tile_x_dm = nullptr;
- int* tile_x_qh = nullptr;
- int* tile_x_sc = nullptr;
-
- allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
-
- __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];
- __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF / QI8_1];
-
- float sum[mmq_y / WARP_SIZE_GGUF][mmq_x / nwarps] = {{0.0f}};
-
- for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
- load_tiles(
- x + row_x_0 * blocks_per_row_x + ib0,
- tile_x_ql,
- tile_x_dm,
- tile_x_qh,
- tile_x_sc,
- threadIdx.y,
- nrows_x - row_x_0 - 1,
- threadIdx.x,
- blocks_per_row_x);
-
-#pragma unroll
- for (int ir = 0; ir < qr && ib0 + ir * blocks_per_warp / qr < blocks_per_row_x; ++ir) {
- const auto kqs = ir * WARP_SIZE_GGUF + threadIdx.x;
- const int kbxd = kqs / QI8_1;
-
-#pragma unroll
- for (int i = 0; i < mmq_x; i += nwarps) {
- const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y - 1); // to prevent out-of-bounds memory accesses
- const block_q8_1* by0 = &y[col_y_eff * blocks_per_col_y + ib0 * (qk / QK8_1) + kbxd];
- const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;
- tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
- }
-
-#pragma unroll
- for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {
- const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE_GGUF / QI8_1)) % mmq_x;
- const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1);
- const int col_y_eff = min(col_y_0 + ids, ncols_y - 1);
-
- // if the sum is not needed it's faster to transform the scale to f32 ahead of time
- const half2* dsi_src =
- &y[col_y_eff * blocks_per_col_y + ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby].ds;
- half2* dsi_dst = &tile_y_ds[ids * (WARP_SIZE_GGUF / QI8_1) + kby];
- if (need_sum) {
- *dsi_dst = *dsi_src;
- } else {
- float* dfi_dst = (float*)dsi_dst;
- *dfi_dst = __low2float(*dsi_src);
- }
- }
-
- __syncthreads();
-
- // #pragma unroll // unrolling this loop causes too much register pressure
- for (int k = ir * WARP_SIZE_GGUF / qr; k < (ir + 1) * WARP_SIZE_GGUF / qr; k += vdr) {
-#pragma unroll
- for (int j = 0; j < mmq_x; j += nwarps) {
-#pragma unroll
- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
- sum[i / WARP_SIZE_GGUF][j / nwarps] += vec_dot(
- tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds, threadIdx.x + i, threadIdx.y + j, k);
- }
- }
- }
- __syncthreads();
- }
- }
-
-#pragma unroll
- for (int j = 0; j < mmq_x; j += nwarps) {
- const auto col_dst = col_dst_0 + j + threadIdx.y;
- if (col_dst >= ncols_dst) {
- return;
- }
-
-#pragma unroll
- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
- const auto row_dst = row_dst_0 + threadIdx.x + i;
- if (row_dst >= nrows_dst) {
- continue;
- }
- dst[col_dst * nrows_dst + row_dst] = sum[i / WARP_SIZE_GGUF][j / nwarps];
- }
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q4_0 64
-#define MMQ_Y_Q4_0 128
-#define NWARPS_Q4_0 8
-#else
-#define MMQ_X_Q4_0 4
-#define MMQ_Y_Q4_0 32
-#define NWARPS_Q4_0 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_0, 2)
-#endif
- mul_mat_q4_0(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q4_0;
- const int mmq_y = MMQ_Y_Q4_0;
- const int nwarps = NWARPS_Q4_0;
-
- mul_mat_q<
- scalar_t,
- QK4_0,
- QR4_0,
- QI4_0,
- true,
- block_q4_0,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q4_0<mmq_y>,
- load_tiles_q4_0<mmq_y, nwarps, need_check>,
- VDR_Q4_0_Q8_1_MMQ,
- vec_dot_q4_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q4_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- int mmq_x = MMQ_X_Q4_0;
- int mmq_y = MMQ_Y_Q4_0;
- int nwarps = NWARPS_Q4_0;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q4_0<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q4_0<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q4_1 64
-#define MMQ_Y_Q4_1 128
-#define NWARPS_Q4_1 8
-#else
-#define MMQ_X_Q4_1 4
-#define MMQ_Y_Q4_1 32
-#define NWARPS_Q4_1 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_1, 2)
-#endif
- mul_mat_q4_1(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q4_1;
- const int mmq_y = MMQ_Y_Q4_1;
- const int nwarps = NWARPS_Q4_1;
-
- mul_mat_q<
- scalar_t,
- QK4_1,
- QR4_1,
- QI4_1,
- true,
- block_q4_1,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q4_1<mmq_y>,
- load_tiles_q4_1<mmq_y, nwarps, need_check>,
- VDR_Q4_1_Q8_1_MMQ,
- vec_dot_q4_1_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q4_1_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- int mmq_x = MMQ_X_Q4_1;
- int mmq_y = MMQ_Y_Q4_1;
- int nwarps = NWARPS_Q4_1;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q4_1<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q4_1<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q5_0 64
-#define MMQ_Y_Q5_0 128
-#define NWARPS_Q5_0 8
-#else
-#define MMQ_X_Q5_0 4
-#define MMQ_Y_Q5_0 32
-#define NWARPS_Q5_0 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_0, 2)
-#endif
- mul_mat_q5_0(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q5_0;
- const int mmq_y = MMQ_Y_Q5_0;
- const int nwarps = NWARPS_Q5_0;
-
- mul_mat_q<
- scalar_t,
- QK5_0,
- QR5_0,
- QI5_0,
- false,
- block_q5_0,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q5_0<mmq_y>,
- load_tiles_q5_0<mmq_y, nwarps, need_check>,
- VDR_Q5_0_Q8_1_MMQ,
- vec_dot_q5_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q5_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- const int mmq_x = MMQ_X_Q5_0;
- const int mmq_y = MMQ_Y_Q5_0;
- const int nwarps = NWARPS_Q5_0;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q5_0<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q5_0<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q5_1 64
-#define MMQ_Y_Q5_1 128
-#define NWARPS_Q5_1 8
-#else
-#define MMQ_X_Q5_1 4
-#define MMQ_Y_Q5_1 32
-#define NWARPS_Q5_1 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_1, 2)
-#endif
- mul_mat_q5_1(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q5_1;
- const int mmq_y = MMQ_Y_Q5_1;
- const int nwarps = NWARPS_Q5_1;
-
- mul_mat_q<
- scalar_t,
- QK5_1,
- QR5_1,
- QI5_1,
- true,
- block_q5_1,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q5_1<mmq_y>,
- load_tiles_q5_1<mmq_y, nwarps, need_check>,
- VDR_Q5_1_Q8_1_MMQ,
- vec_dot_q5_1_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q5_1_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- const int mmq_x = MMQ_X_Q5_1;
- const int mmq_y = MMQ_Y_Q5_1;
- const int nwarps = NWARPS_Q5_1;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q5_1<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q5_1<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q8_0 64
-#define MMQ_Y_Q8_0 128
-#define NWARPS_Q8_0 8
-#else
-#define MMQ_X_Q8_0 4
-#define MMQ_Y_Q8_0 32
-#define NWARPS_Q8_0 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q8_0, 2)
-#endif
- mul_mat_q8_0(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q8_0;
- const int mmq_y = MMQ_Y_Q8_0;
- const int nwarps = NWARPS_Q8_0;
-
- mul_mat_q<
- scalar_t,
- QK8_0,
- QR8_0,
- QI8_0,
- false,
- block_q8_0,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q8_0<mmq_y>,
- load_tiles_q8_0<mmq_y, nwarps, need_check>,
- VDR_Q8_0_Q8_1_MMQ,
- vec_dot_q8_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q8_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- const int mmq_x = MMQ_X_Q8_0;
- const int mmq_y = MMQ_Y_Q8_0;
- const int nwarps = NWARPS_Q8_0;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q8_0<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q8_0<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q2_K 64
-#define MMQ_Y_Q2_K 128
-#define NWARPS_Q2_K 8
-#else
-#define MMQ_X_Q2_K 4
-#define MMQ_Y_Q2_K 32
-#define NWARPS_Q2_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q2_K, 2)
-#endif
- mul_mat_q2_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q2_K;
- const int mmq_y = MMQ_Y_Q2_K;
- const int nwarps = NWARPS_Q2_K;
-
- mul_mat_q<
- scalar_t,
- QK_K,
- QR2_K,
- QI2_K,
- false,
- block_q2_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q2_K<mmq_y>,
- load_tiles_q2_K<mmq_y, nwarps, need_check>,
- VDR_Q2_K_Q8_1_MMQ,
- vec_dot_q2_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q2_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- const int mmq_x = MMQ_X_Q2_K;
- const int mmq_y = MMQ_Y_Q2_K;
- const int nwarps = NWARPS_Q2_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q2_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q2_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q3_K 64
-#define MMQ_Y_Q3_K 128
-#define NWARPS_Q3_K 8
-#else
-#define MMQ_X_Q3_K 4
-#define MMQ_Y_Q3_K 32
-#define NWARPS_Q3_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q3_K, 2)
-#endif
- mul_mat_q3_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
-
- const int mmq_x = MMQ_X_Q3_K;
- const int mmq_y = MMQ_Y_Q3_K;
- const int nwarps = NWARPS_Q3_K;
-
- mul_mat_q<
- scalar_t,
- QK_K,
- QR3_K,
- QI3_K,
- false,
- block_q3_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q3_K<mmq_y>,
- load_tiles_q3_K<mmq_y, nwarps, need_check>,
- VDR_Q3_K_Q8_1_MMQ,
- vec_dot_q3_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q3_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- const int mmq_x = MMQ_X_Q3_K;
- const int mmq_y = MMQ_Y_Q3_K;
- const int nwarps = NWARPS_Q3_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q3_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q3_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q4_K 64
-#define MMQ_Y_Q4_K 128
-#define NWARPS_Q4_K 8
-#else
-#define MMQ_X_Q4_K 4
-#define MMQ_Y_Q4_K 32
-#define NWARPS_Q4_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_K, 2)
-#endif
- mul_mat_q4_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q4_K;
- const int mmq_y = MMQ_Y_Q4_K;
- const int nwarps = NWARPS_Q4_K;
-
- mul_mat_q<
- scalar_t,
- QK_K,
- QR4_K,
- QI4_K,
- true,
- block_q4_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q4_K<mmq_y>,
- load_tiles_q4_K<mmq_y, nwarps, need_check>,
- VDR_Q4_K_Q8_1_MMQ,
- vec_dot_q4_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q4_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- const int mmq_x = MMQ_X_Q4_K;
- const int mmq_y = MMQ_Y_Q4_K;
- const int nwarps = NWARPS_Q4_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q4_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q4_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q5_K 64
-#define MMQ_Y_Q5_K 128
-#define NWARPS_Q5_K 8
-#else
-#define MMQ_X_Q5_K 4
-#define MMQ_Y_Q5_K 32
-#define NWARPS_Q5_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_K, 2)
-#endif
- mul_mat_q5_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q5_K;
- const int mmq_y = MMQ_Y_Q5_K;
- const int nwarps = NWARPS_Q5_K;
-
- mul_mat_q<
- scalar_t,
- QK_K,
- QR5_K,
- QI5_K,
- true,
- block_q5_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q5_K<mmq_y>,
- load_tiles_q5_K<mmq_y, nwarps, need_check>,
- VDR_Q5_K_Q8_1_MMQ,
- vec_dot_q5_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q5_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- const int mmq_x = MMQ_X_Q5_K;
- const int mmq_y = MMQ_Y_Q5_K;
- const int nwarps = NWARPS_Q5_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q5_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q5_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
-
-#if defined(USE_ROCM)
-#define MMQ_X_Q6_K 64
-#define MMQ_Y_Q6_K 128
-#define NWARPS_Q6_K 8
-#else
-#define MMQ_X_Q6_K 4
-#define MMQ_Y_Q6_K 32
-#define NWARPS_Q6_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q6_K, 2)
-#endif
- mul_mat_q6_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst) {
- const int mmq_x = MMQ_X_Q6_K;
- const int mmq_y = MMQ_Y_Q6_K;
- const int nwarps = NWARPS_Q6_K;
-
- mul_mat_q<
- scalar_t,
- QK_K,
- QR6_K,
- QI6_K,
- false,
- block_q6_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q6_K<mmq_y>,
- load_tiles_q6_K<mmq_y, nwarps, need_check>,
- VDR_Q6_K_Q8_1_MMQ,
- vec_dot_q6_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
-}
-
-template <typename scalar_t>
-static void ggml_mul_mat_q6_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- cudaStream_t stream) {
- const int mmq_x = MMQ_X_Q6_K;
- const int mmq_y = MMQ_Y_Q6_K;
- const int nwarps = NWARPS_Q6_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- const bool need_check = false;
- mul_mat_q6_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- } else {
- const bool need_check = true;
- mul_mat_q6_K<scalar_t, need_check>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
- }
-}
diff --git a/python/freetoken/kernel/csrc/gguf/mmvq.cuh b/python/freetoken/kernel/csrc/gguf/mmvq.cuh
deleted file mode 100644
index 7331731a..00000000
--- a/python/freetoken/kernel/csrc/gguf/mmvq.cuh
+++ /dev/null
@@ -1,352 +0,0 @@
-// copied from
-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/mmvq.cuh
-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu
-template <typename scalar_t, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>
-static __global__ void mul_mat_vec_q(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int ncols,
- const int nrows,
- const int nvecs) {
- const auto row = blockIdx.x * blockDim.y + threadIdx.y;
- const auto vec = blockIdx.y;
-
- if (row >= nrows || vec >= nvecs) {
- return;
- }
-
- const int blocks_per_row = ncols / qk;
- const int blocks_per_warp = vdr * WARP_SIZE / qi;
- const int nrows_y = (ncols + 512 - 1) / 512 * 512;
-
- // partial sum for each thread
- float tmp = 0.0f;
-
- const block_q_t* x = (const block_q_t*)vx;
- const block_q8_1* y = (const block_q8_1*)vy;
-
- for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) {
- const int ibx = row * blocks_per_row + i; // x block index
-
- const int iby = vec * (nrows_y / QK8_1) + i * (qk / QK8_1); // y block index that aligns with ibx
-
- const int iqs = vdr * (threadIdx.x % (qi / vdr)); // x block quant index when casting the quants to int
-
- tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);
- }
-
- // sum up partial sums and write back result
-#pragma unroll
- for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
- tmp += SGLANG_SHFL_XOR_SYNC(uint32_t(-1), tmp, mask);
- }
-
- if (threadIdx.x == 0) {
- dst[vec * nrows + row] = tmp;
- }
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q4_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q4_1_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q5_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q5_1_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q8_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q2_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q3_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q4_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q5_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_q6_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq2_xxs_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq2_xs_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq2_s_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq3_xxs_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq1_s_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq1_m_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI1_M, block_iq1_m, 1, vec_dot_iq1_m_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq4_nl_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq4_xs_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
-
-template <typename scalar_t>
-static void mul_mat_vec_iq3_s_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int ncols,
- const int nrows,
- const int nvecs,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, nvecs, 1);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- mul_mat_vec_q<scalar_t, QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
-}
diff --git a/python/freetoken/kernel/csrc/gguf/moe.cuh b/python/freetoken/kernel/csrc/gguf/moe.cuh
deleted file mode 100644
index 91e434fa..00000000
--- a/python/freetoken/kernel/csrc/gguf/moe.cuh
+++ /dev/null
@@ -1,1379 +0,0 @@
-// copied from
-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/moe.cuh
-#include <cstdint>
-
-/* Adapted from ./csrc/quantization/gguf/mmq.cuh
- */
-template <
- typename scalar_t,
- int qk,
- int qr,
- int qi,
- bool need_sum,
- typename block_q_t,
- int mmq_x,
- int mmq_y,
- int nwarps,
- allocate_tiles_cuda_t allocate_tiles,
- load_tiles_cuda_t load_tiles,
- int vdr,
- vec_dot_q_mul_mat_cuda_t vec_dot>
-static __device__ __forceinline__ void moe_q(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* __restrict__ sorted_token_ids,
- const int* __restrict__ expert_ids,
- const int* __restrict__ num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int blocks_per_row_x = ncols_x / qk;
- const int blocks_per_col_y = nrows_y / QK8_1;
- const int blocks_per_warp = WARP_SIZE_GGUF / qi;
-
- const int ncols_dst = ncols_y * top_k;
-
- const auto row_dst_0 = blockIdx.x * mmq_y;
- const int& row_x_0 = row_dst_0;
-
- const auto col_dst_0 = blockIdx.y * mmq_x;
-
- int token_offs[mmq_x / nwarps];
- for (int i = 0; i < mmq_x; i += nwarps) {
- token_offs[i / nwarps] = sorted_token_ids[col_dst_0 + threadIdx.y + i];
- }
-
- const int exp_idx = expert_ids[blockIdx.y];
- if (exp_idx > 255 || exp_idx < 0) return;
- if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
-
- const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
- const block_q8_1* y = (const block_q8_1*)(vy);
-
- int* tile_x_ql = nullptr;
- half2* tile_x_dm = nullptr;
- int* tile_x_qh = nullptr;
- int* tile_x_sc = nullptr;
-
- allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
-
- __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];
- __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF / QI8_1];
-
- float sum[mmq_y / WARP_SIZE_GGUF][mmq_x / nwarps] = {{0.0f}};
-
- for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
- load_tiles(
- x + row_x_0 * blocks_per_row_x + ib0,
- tile_x_ql,
- tile_x_dm,
- tile_x_qh,
- tile_x_sc,
- threadIdx.y,
- nrows_x - row_x_0 - 1,
- threadIdx.x,
- blocks_per_row_x);
-
- const int n_per_r = ((qk * blocks_per_warp) / qr);
-#pragma unroll
- for (int ir = 0; ir < qr && ib0 * qk + ir * n_per_r < ncols_x; ++ir) {
- const auto kqs = ir * WARP_SIZE_GGUF + threadIdx.x;
- const int kbxd = kqs / QI8_1;
-
-#pragma unroll
- for (int i = 0; i < mmq_x; i += nwarps) {
- const int col_y_eff = token_offs[i / nwarps] / top_k;
- const int block_x = ib0 * (qk / QK8_1) + kbxd;
- if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {
- const block_q8_1* by0 = &y[col_y_eff * blocks_per_col_y + block_x];
- const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;
- tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
- }
- }
-
- if (threadIdx.x < n_per_r / QK8_1) {
- const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1);
- const int col_y_eff = token_offs[threadIdx.y] / top_k;
- const int block_x = ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby;
-
- if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {
- const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds;
- half2* dsi_dst = &tile_y_ds[threadIdx.y * (WARP_SIZE_GGUF / QI8_1) + kby];
-
- if (need_sum) {
- *dsi_dst = *dsi_src;
- } else {
- float* dfi_dst = (float*)dsi_dst;
- *dfi_dst = __low2float(*dsi_src);
- }
- }
- }
- __syncthreads();
-
- // #pragma unroll // unrolling this loop causes too much register pressure
- for (int k = ir * WARP_SIZE_GGUF / qr; k < (ir + 1) * WARP_SIZE_GGUF / qr; k += vdr) {
-#pragma unroll
- for (int j = 0; j < mmq_x; j += nwarps) {
-#pragma unroll
- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
- sum[i / WARP_SIZE_GGUF][j / nwarps] += vec_dot(
- tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds, threadIdx.x + i, threadIdx.y + j, k);
- }
- }
- }
- __syncthreads();
- }
- }
-
-#pragma unroll
- for (int j = 0; j < mmq_x; j += nwarps) {
- const int col_dst = token_offs[j / nwarps];
- if (col_dst >= ncols_dst) {
- return;
- }
-
-#pragma unroll
- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
- const auto row_dst = row_dst_0 + threadIdx.x + i;
- if (row_dst >= nrows_dst) {
- continue;
- }
- dst[col_dst * nrows_dst + row_dst] = sum[i / WARP_SIZE_GGUF][j / nwarps];
- }
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q4_0 8
-#define MOE_Y_Q4_0 128
-#define NWARPS_Q4_0 8
-#else
-#define MOE_X_Q4_0 4
-#define MOE_Y_Q4_0 32
-#define NWARPS_Q4_0 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_0, 2)
-#endif
- moe_q4_0(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q4_0;
- const int mmq_y = MOE_Y_Q4_0;
- const int nwarps = NWARPS_Q4_0;
-
- moe_q<
- scalar_t,
- QK4_0,
- QR4_0,
- QI4_0,
- true,
- block_q4_0,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q4_0<mmq_y>,
- load_tiles_q4_0<mmq_y, nwarps, need_check>,
- VDR_Q4_0_Q8_1_MMQ,
- vec_dot_q4_0_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q4_0_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- int mmq_x = MOE_X_Q4_0;
- int mmq_y = MOE_Y_Q4_0;
- int nwarps = NWARPS_Q4_0;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q4_1 8
-#define MOE_Y_Q4_1 128
-#define NWARPS_Q4_1 8
-#else
-#define MOE_X_Q4_1 4
-#define MOE_Y_Q4_1 32
-#define NWARPS_Q4_1 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_1, 2)
-#endif
- moe_q4_1(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q4_1;
- const int mmq_y = MOE_Y_Q4_1;
- const int nwarps = NWARPS_Q4_1;
-
- moe_q<
- scalar_t,
- QK4_1,
- QR4_1,
- QI4_1,
- true,
- block_q4_1,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q4_1<mmq_y>,
- load_tiles_q4_1<mmq_y, nwarps, need_check>,
- VDR_Q4_1_Q8_1_MMQ,
- vec_dot_q4_1_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q4_1_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- int mmq_x = MOE_X_Q4_1;
- int mmq_y = MOE_Y_Q4_1;
- int nwarps = NWARPS_Q4_1;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q5_0 8
-#define MOE_Y_Q5_0 128
-#define NWARPS_Q5_0 8
-#else
-#define MOE_X_Q5_0 4
-#define MOE_Y_Q5_0 32
-#define NWARPS_Q5_0 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_0, 2)
-#endif
- moe_q5_0(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q5_0;
- const int mmq_y = MOE_Y_Q5_0;
- const int nwarps = NWARPS_Q5_0;
-
- moe_q<
- scalar_t,
- QK5_0,
- QR5_0,
- QI5_0,
- false,
- block_q5_0,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q5_0<mmq_y>,
- load_tiles_q5_0<mmq_y, nwarps, need_check>,
- VDR_Q5_0_Q8_1_MMQ,
- vec_dot_q5_0_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q5_0_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- const int mmq_x = MOE_X_Q5_0;
- const int mmq_y = MOE_Y_Q5_0;
- const int nwarps = NWARPS_Q5_0;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q5_1 8
-#define MOE_Y_Q5_1 128
-#define NWARPS_Q5_1 8
-#else
-#define MOE_X_Q5_1 4
-#define MOE_Y_Q5_1 32
-#define NWARPS_Q5_1 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_1, 2)
-#endif
- moe_q5_1(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q5_1;
- const int mmq_y = MOE_Y_Q5_1;
- const int nwarps = NWARPS_Q5_1;
-
- moe_q<
- scalar_t,
- QK5_1,
- QR5_1,
- QI5_1,
- true,
- block_q5_1,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q5_1<mmq_y>,
- load_tiles_q5_1<mmq_y, nwarps, need_check>,
- VDR_Q5_1_Q8_1_MMQ,
- vec_dot_q5_1_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q5_1_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- const int mmq_x = MOE_X_Q5_1;
- const int mmq_y = MOE_Y_Q5_1;
- const int nwarps = NWARPS_Q5_1;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q8_0 8
-#define MOE_Y_Q8_0 128
-#define NWARPS_Q8_0 8
-#else
-#define MOE_X_Q8_0 4
-#define MOE_Y_Q8_0 32
-#define NWARPS_Q8_0 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q8_0, 2)
-#endif
- moe_q8_0(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q8_0;
- const int mmq_y = MOE_Y_Q8_0;
- const int nwarps = NWARPS_Q8_0;
-
- moe_q<
- scalar_t,
- QK8_0,
- QR8_0,
- QI8_0,
- false,
- block_q8_0,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q8_0<mmq_y>,
- load_tiles_q8_0<mmq_y, nwarps, need_check>,
- VDR_Q8_0_Q8_1_MMQ,
- vec_dot_q8_0_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q8_0_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- const int mmq_x = MOE_X_Q8_0;
- const int mmq_y = MOE_Y_Q8_0;
- const int nwarps = NWARPS_Q8_0;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q2_K 8
-#define MOE_Y_Q2_K 128
-#define NWARPS_Q2_K 8
-#else
-#define MOE_X_Q2_K 4
-#define MOE_Y_Q2_K 32
-#define NWARPS_Q2_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q2_K, 2)
-#endif
- moe_q2_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q2_K;
- const int mmq_y = MOE_Y_Q2_K;
- const int nwarps = NWARPS_Q2_K;
-
- moe_q<
- scalar_t,
- QK_K,
- QR2_K,
- QI2_K,
- false,
- block_q2_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q2_K<mmq_y>,
- load_tiles_q2_K<mmq_y, nwarps, need_check>,
- VDR_Q2_K_Q8_1_MMQ,
- vec_dot_q2_K_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q2_K_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- const int mmq_x = MOE_X_Q2_K;
- const int mmq_y = MOE_Y_Q2_K;
- const int nwarps = NWARPS_Q2_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q3_K 8
-#define MOE_Y_Q3_K 128
-#define NWARPS_Q3_K 8
-#else
-#define MOE_X_Q3_K 4
-#define MOE_Y_Q3_K 32
-#define NWARPS_Q3_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q3_K, 2)
-#endif
- moe_q3_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
-
- const int mmq_x = MOE_X_Q3_K;
- const int mmq_y = MOE_Y_Q3_K;
- const int nwarps = NWARPS_Q3_K;
-
- moe_q<
- scalar_t,
- QK_K,
- QR3_K,
- QI3_K,
- false,
- block_q3_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q3_K<mmq_y>,
- load_tiles_q3_K<mmq_y, nwarps, need_check>,
- VDR_Q3_K_Q8_1_MMQ,
- vec_dot_q3_K_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-template <typename scalar_t>
-static void ggml_moe_q3_K_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- const int mmq_x = MOE_X_Q3_K;
- const int mmq_y = MOE_Y_Q3_K;
- const int nwarps = NWARPS_Q3_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q4_K 8
-#define MOE_Y_Q4_K 128
-#define NWARPS_Q4_K 8
-#else
-#define MOE_X_Q4_K 4
-#define MOE_Y_Q4_K 32
-#define NWARPS_Q4_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_K, 2)
-#endif
- moe_q4_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q4_K;
- const int mmq_y = MOE_Y_Q4_K;
- const int nwarps = NWARPS_Q4_K;
-
- moe_q<
- scalar_t,
- QK_K,
- QR4_K,
- QI4_K,
- true,
- block_q4_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q4_K<mmq_y>,
- load_tiles_q4_K<mmq_y, nwarps, need_check>,
- VDR_Q4_K_Q8_1_MMQ,
- vec_dot_q4_K_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q4_K_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- const int mmq_x = MOE_X_Q4_K;
- const int mmq_y = MOE_Y_Q4_K;
- const int nwarps = NWARPS_Q4_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q5_K 8
-#define MOE_Y_Q5_K 128
-#define NWARPS_Q5_K 8
-#else
-#define MOE_X_Q5_K 4
-#define MOE_Y_Q5_K 32
-#define NWARPS_Q5_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_K, 2)
-#endif
- moe_q5_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q5_K;
- const int mmq_y = MOE_Y_Q5_K;
- const int nwarps = NWARPS_Q5_K;
-
- moe_q<
- scalar_t,
- QK_K,
- QR5_K,
- QI5_K,
- true,
- block_q5_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q5_K<mmq_y>,
- load_tiles_q5_K<mmq_y, nwarps, need_check>,
- VDR_Q5_K_Q8_1_MMQ,
- vec_dot_q5_K_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q5_K_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- const int mmq_x = MOE_X_Q5_K;
- const int mmq_y = MOE_Y_Q5_K;
- const int nwarps = NWARPS_Q5_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
-
-#if defined(USE_ROCM)
-#define MOE_X_Q6_K 8
-#define MOE_Y_Q6_K 128
-#define NWARPS_Q6_K 8
-#else
-#define MOE_X_Q6_K 4
-#define MOE_Y_Q6_K 32
-#define NWARPS_Q6_K 4
-#endif
-
-template <typename scalar_t, bool need_check>
-static __global__ void
-#if defined(USE_ROCM)
-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q6_K, 2)
-#endif
- moe_q6_K(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k) {
- const int mmq_x = MOE_X_Q6_K;
- const int mmq_y = MOE_Y_Q6_K;
- const int nwarps = NWARPS_Q6_K;
-
- moe_q<
- scalar_t,
- QK_K,
- QR6_K,
- QI6_K,
- false,
- block_q6_K,
- mmq_x,
- mmq_y,
- nwarps,
- allocate_tiles_q6_K<mmq_y>,
- load_tiles_q6_K<mmq_y, nwarps, need_check>,
- VDR_Q6_K_Q8_1_MMQ,
- vec_dot_q6_K_q8_1_mul_mat>(
- vx,
- vy,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
-}
-
-template <typename scalar_t>
-static void ggml_moe_q6_K_q8_1_cuda(
- const void* inp,
- const void* w,
- scalar_t* dst,
- const int* sorted_token_ids,
- const int* expert_ids,
- const int* num_tokens_post_padded,
- const int exp_stride,
- const int ncols_x,
- const int nrows_x,
- const int ncols_y,
- const int nrows_y,
- const int nrows_dst,
- const int top_k,
- const int tokens_post_padded,
- cudaStream_t stream) {
- const int mmq_x = MOE_X_Q6_K;
- const int mmq_y = MOE_Y_Q6_K;
- const int nwarps = NWARPS_Q6_K;
-
- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
- const int block_num_y = (tokens_post_padded) / mmq_x;
- const dim3 block_nums(block_num_x, block_num_y, 1);
- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
-
- if (nrows_x % mmq_y == 0) {
- constexpr bool need_check = false;
- moe_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- } else {
- constexpr bool need_check = true;
- moe_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
- w,
- inp,
- dst,
- sorted_token_ids,
- expert_ids,
- num_tokens_post_padded,
- exp_stride,
- ncols_x,
- nrows_x,
- ncols_y,
- nrows_y,
- nrows_dst,
- top_k);
- }
-}
diff --git a/python/freetoken/kernel/csrc/gguf/moe_vec.cuh b/python/freetoken/kernel/csrc/gguf/moe_vec.cuh
deleted file mode 100644
index 8cef9e08..00000000
--- a/python/freetoken/kernel/csrc/gguf/moe_vec.cuh
+++ /dev/null
@@ -1,413 +0,0 @@
-// copied from
-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/moe_vec.cuh
-// copied and adapted from
-// https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu
-template <typename scalar_t, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>
-static __global__ void moe_vec_q(
- const void* __restrict__ vx,
- const void* __restrict__ vy,
- scalar_t* __restrict__ dst,
- const int* topk_ids,
- const int topk,
- const int ncols,
- const int nrows,
- const int token_stride) {
- const auto row = blockIdx.x * blockDim.y + threadIdx.y;
-
- const auto token = blockIdx.z / topk;
- const auto expert = (topk_ids)[blockIdx.z];
-
- if (row >= nrows) {
- return;
- }
-
- const int blocks_per_row = ncols / qk;
- const int blocks_per_warp = vdr * WARP_SIZE / qi;
-
- // partial sum for each thread
- float tmp = 0.0f;
-
- const block_q_t* x = ((const block_q_t*)vx) + expert * nrows * blocks_per_row;
- const block_q8_1* y = (const block_q8_1*)(((const int*)vy) + token * token_stride);
-
- for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) {
- const int ibx = row * blocks_per_row + i; // x block index
-
- const int iby = i * (qk / QK8_1); // y block index that aligns with ibx
-
- const int iqs = vdr * (threadIdx.x % (qi / vdr)); // x block quant index when casting the quants to int
-
- tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);
- }
-
- // sum up partial sums and write back result
-#pragma unroll
- for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
- tmp += SGLANG_SHFL_XOR_SYNC(uint32_t(-1), tmp, mask);
- }
-
- if (threadIdx.x == 0) {
- dst[blockIdx.z * nrows + row] = tmp;
- }
-}
-
-template <typename scalar_t>
-static void moe_vec_q4_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q4_1_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q5_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q5_1_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q8_0_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q2_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q3_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q4_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q5_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_q6_K_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq2_xxs_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq2_xs_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq2_s_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq3_xxs_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq1_s_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq1_m_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI1_M, block_iq1_m, 1, vec_dot_iq1_m_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq4_nl_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq4_xs_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
-
-template <typename scalar_t>
-static void moe_vec_iq3_s_q8_1_cuda(
- const void* vx,
- const void* vy,
- scalar_t* dst,
- const int* topk_ids,
- const int top_k,
- const int tokens,
- const int ncols,
- const int nrows,
- const int token_stride,
- cudaStream_t stream) {
- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
- const dim3 block_nums(block_num_y, 1, tokens * top_k);
- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
- moe_vec_q<scalar_t, QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>
- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
-}
diff --git a/python/freetoken/kernel/csrc/gguf/vecdotq.cuh b/python/freetoken/kernel/csrc/gguf/vecdotq.cuh
deleted file mode 100644
index 08b4cd26..00000000
--- a/python/freetoken/kernel/csrc/gguf/vecdotq.cuh
+++ /dev/null
@@ -1,2037 +0,0 @@
-// copied from
-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/vecdotq.cuh
-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/vecdotq.cuh
-// and https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
-static __device__ __forceinline__ int get_int_b2(const void* x, const int& i32) {
- const uint16_t* x16 = (const uint16_t*)x; // assume at least 2 byte alignment
-
- int x32 = x16[2 * i32 + 0] << 0;
- x32 |= x16[2 * i32 + 1] << 16;
-
- return x32;
-}
-
-static __device__ __forceinline__ int get_int_b4(const void* x, const int& i32) {
- return ((const int*)x)[i32]; // assume at least 4 byte alignment
-}
-
-static __device__ __forceinline__ int get_int_from_int8(const int8_t* x8, const int& i32) {
- const uint16_t* x16 = (const uint16_t*)(x8 + sizeof(int) * i32); // assume at least 2 byte alignment
- int x32 = 0;
- x32 |= x16[0] << 0;
- x32 |= x16[1] << 16;
- return x32;
-}
-
-static __device__ __forceinline__ int get_int_from_uint8(const uint8_t* x8, const int& i32) {
- const uint16_t* x16 = (const uint16_t*)(x8 + sizeof(int) * i32); // assume at least 2 byte alignment
- int x32 = 0;
- x32 |= x16[0] << 0;
- x32 |= x16[1] << 16;
- return x32;
-}
-
-static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t* x8, const int& i32) {
- return *((const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
-}
-
-static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t* x8, const int& i32) {
- return *((const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
-}
-
-// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called
-// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q
-
-#define VDR_Q4_0_Q8_1_MMVQ 2
-#define VDR_Q4_0_Q8_1_MMQ 4
-
-template <int vdr>
-static __device__ __forceinline__ float
-vec_dot_q4_0_q8_1_impl(const int* v, const int* u, const float& d4, const half2& ds8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- int sumi = 0;
-
-#pragma unroll
- for (int i = 0; i < vdr; ++i) {
- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
-
- // SIMD dot product of quantized values
- sumi = __dp4a(vi0, u[2 * i + 0], sumi);
- sumi = __dp4a(vi1, u[2 * i + 1], sumi);
- }
-
- const float2 ds8f = __half22float2(ds8);
-
- // second part effectively subtracts 8 from each quant value
- return d4 * (sumi * ds8f.x - (8 * vdr / QI4_0) * ds8f.y);
-#endif
-}
-
-#define VDR_Q4_1_Q8_1_MMVQ 2
-#define VDR_Q4_1_Q8_1_MMQ 4
-
-template <int vdr>
-static __device__ __forceinline__ float
-vec_dot_q4_1_q8_1_impl(const int* v, const int* u, const half2& dm4, const half2& ds8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- int sumi = 0;
-
-#pragma unroll
- for (int i = 0; i < vdr; ++i) {
- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
-
- // SIMD dot product of quantized values
- sumi = __dp4a(vi0, u[2 * i + 0], sumi);
- sumi = __dp4a(vi1, u[2 * i + 1], sumi);
- }
-
- const float2 tmp = __half22float2(__hmul2(dm4, ds8));
- const float d4d8 = tmp.x;
- const float m4s8 = tmp.y;
-
- // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
- return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
-#endif
-}
-
-#define VDR_Q5_0_Q8_1_MMVQ 2
-#define VDR_Q5_0_Q8_1_MMQ 4
-
-template <int vdr>
-static __device__ __forceinline__ float
-vec_dot_q5_0_q8_1_impl(const int* vl, const int* vh, const int* u, const float& d5, const half2& ds8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- int sumi = 0;
-
-#pragma unroll
- for (int i = 0; i < vdr; ++i) {
- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
- sumi = __dp4a(vi0, u[2 * i + 0], sumi); // SIMD dot product of quantized values
-
- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
- sumi = __dp4a(vi1, u[2 * i + 1], sumi); // SIMD dot product of quantized values
- }
-
- const float2 ds8f = __half22float2(ds8);
-
- // second part effectively subtracts 16 from each quant value
- return d5 * (sumi * ds8f.x - (16 * vdr / QI5_0) * ds8f.y);
-#endif
-}
-
-#define VDR_Q5_1_Q8_1_MMVQ 2
-#define VDR_Q5_1_Q8_1_MMQ 4
-
-template <int vdr>
-static __device__ __forceinline__ float
-vec_dot_q5_1_q8_1_impl(const int* vl, const int* vh, const int* u, const half2& dm5, const half2& ds8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- int sumi = 0;
-
-#pragma unroll
- for (int i = 0; i < vdr; ++i) {
- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
- sumi = __dp4a(vi0, u[2 * i + 0], sumi); // SIMD dot product of quantized values
-
- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
- sumi = __dp4a(vi1, u[2 * i + 1], sumi); // SIMD dot product of quantized values
- }
-
- const float2 tmp = __half22float2(__hmul2(dm5, ds8));
- const float d5d8 = tmp.x;
- const float m5s8 = tmp.y;
-
- // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
- return sumi * d5d8 + m5s8 / (QI5_1 / vdr);
-#endif
-}
-
-#define VDR_Q8_0_Q8_1_MMVQ 2
-#define VDR_Q8_0_Q8_1_MMQ 8
-
-template <int vdr>
-static __device__ __forceinline__ float
-vec_dot_q8_0_q8_1_impl(const int* v, const int* u, const float& d8_0, const float& d8_1) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- int sumi = 0;
-
-#pragma unroll
- for (int i = 0; i < vdr; ++i) {
- // SIMD dot product of quantized values
- sumi = __dp4a(v[i], u[i], sumi);
- }
- return d8_0 * d8_1 * sumi;
-#endif
-}
-
-template <int vdr>
-static __device__ __forceinline__ float
-vec_dot_q8_1_q8_1_impl(const int* v, const int* u, const half2& dm8, const half2& ds8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
-
- int sumi = 0;
-
-#pragma unroll
- for (int i = 0; i < vdr; ++i) {
- // SIMD dot product of quantized values
- sumi = __dp4a(v[i], u[i], sumi);
- }
-
- const float2 tmp = __half22float2(__hmul2(dm8, ds8));
- const float d8d8 = tmp.x;
- const float m8s8 = tmp.y;
-
- // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
- return sumi * d8d8 + m8s8 / (QI8_1 / vdr);
-#endif
-}
-
-#define VDR_Q2_K_Q8_1_MMVQ 1
-#define VDR_Q2_K_Q8_1_MMQ 2
-
-// contiguous v/x values
-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
- const int& v,
- const int* __restrict__ u,
- const uint8_t* __restrict__ scales,
- const half2& dm2,
- const float* __restrict__ d8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- float sumf_d = 0.0f;
- float sumf_m = 0.0f;
-
-#pragma unroll
- for (int i = 0; i < QR2_K; ++i) {
- const int sc = scales[2 * i];
-
- const int vi = (v >> (2 * i)) & 0x03030303;
-
- sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
-
- // fill int with 4x m
- int m = sc >> 4;
- m |= m << 8;
- m |= m << 16;
- sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
- }
-
- const float2 dm2f = __half22float2(dm2);
-
- return dm2f.x * sumf_d - dm2f.y * sumf_m;
-#endif
-}
-
-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
- const int* __restrict__ v,
- const int* __restrict__ u,
- const uint8_t* __restrict__ scales,
- const half2& dm2,
- const float& d8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- int sumi_d = 0;
- int sumi_m = 0;
-
-#pragma unroll
- for (int i0 = 0; i0 < QI8_1; i0 += QI8_1 / 2) {
- int sumi_d_sc = 0;
-
- const int sc = scales[i0 / (QI8_1 / 2)];
-
- // fill int with 4x m
- int m = sc >> 4;
- m |= m << 8;
- m |= m << 16;
-
-#pragma unroll
- for (int i = i0; i < i0 + QI8_1 / 2; ++i) {
- sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product
- sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m
- }
-
- sumi_d += sumi_d_sc * (sc & 0xF);
- }
-
- const float2 dm2f = __half22float2(dm2);
-
- return d8 * (dm2f.x * sumi_d - dm2f.y * sumi_m);
-#endif
-}
-
-#define VDR_Q3_K_Q8_1_MMVQ 1
-#define VDR_Q3_K_Q8_1_MMQ 2
-
-// contiguous v/x values
-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
- const int& vl,
- const int& vh,
- const int* __restrict__ u,
- const uint8_t* __restrict__ scales,
- const int& scale_offset,
- const float& d3,
- const float* __restrict__ d8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
-
- float sumf = 0.0f;
-
-#pragma unroll
- for (int i = 0; i < QR3_K; ++i) {
- const int isc = scale_offset + 2 * i;
-
- const int isc_low = isc % (QK_K / 32);
- const int sc_shift_low = 4 * (isc / (QK_K / 32));
- const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
-
- const int isc_high = isc % (QK_K / 64);
- const int sc_shift_high = 2 * (isc / (QK_K / 64));
- const int sc_high = ((scales[(QK_K / 32) + isc_high] >> sc_shift_high) & 3) << 4;
-
- const int sc = (sc_low | sc_high) - 32;
-
- const int vil = (vl >> (2 * i)) & 0x03030303;
-
- const int vih = ((vh >> i) << 2) & 0x04040404;
-
- const int vi = __vsubss4(vil, vih);
-
- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product
- }
-
- return d3 * sumf;
-#endif
-}
-
-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
- const int* __restrict__ v,
- const int* __restrict__ u,
- const int8_t* __restrict__ scales,
- const float& d3,
- const float& d8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- int sumi = 0;
-
-#pragma unroll
- for (int i0 = 0; i0 < QR3_K * VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1 / 2) {
- int sumi_sc = 0;
-
- for (int i = i0; i < i0 + QI8_1 / 2; ++i) {
- sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product
- }
-
- sumi += sumi_sc * scales[i0 / (QI8_1 / 2)];
- }
-
- return d3 * d8 * sumi;
-#endif
-}
-
-#define VDR_Q4_K_Q8_1_MMVQ 2
-#define VDR_Q4_K_Q8_1_MMQ 8
-
-// contiguous v/x values
-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
- const int* __restrict__ v,
- const int* __restrict__ u,
- const uint8_t* __restrict__ sc,
- const uint8_t* __restrict__ m,
- const half2& dm4,
- const float* __restrict__ d8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
-
- float sumf_d = 0.0f;
- float sumf_m = 0.0f;
-
-#pragma unroll
- for (int i = 0; i < QR4_K; ++i) {
- const int v0i = (v[0] >> (4 * i)) & 0x0F0F0F0F;
- const int v1i = (v[1] >> (4 * i)) & 0x0F0F0F0F;
-
- const int dot1 = __dp4a(v1i, u[2 * i + 1], __dp4a(v0i, u[2 * i + 0], 0)); // SIMD dot product
- const int dot2 = __dp4a(0x01010101, u[2 * i + 1], __dp4a(0x01010101, u[2 * i + 0], 0)); // sum of u
-
- sumf_d += d8[i] * (dot1 * sc[i]);
- sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
- }
-
- const float2 dm4f = __half22float2(dm4);
- return dm4f.x * sumf_d - dm4f.y * sumf_m;
-#endif
-}
-
-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
- const int* __restrict__ v,
- const int* __restrict__ u,
- const uint8_t* __restrict__ sc,
- const uint8_t* __restrict__ m,
- const half2& dm4,
- const half2* __restrict__ ds8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- float sumf_d = 0.0f;
- float sumf_m = 0.0f;
-
-#pragma unroll
- for (int i = 0; i < QR4_K * VDR_Q4_K_Q8_1_MMQ / QI8_1; ++i) {
- int sumi_d = 0;
-
-#pragma unroll
- for (int j = 0; j < QI8_1; ++j) {
- sumi_d = __dp4a((v[j] >> (4 * i)) & 0x0F0F0F0F, u[i * QI8_1 + j], sumi_d); // SIMD dot product
- }
-
- const float2 ds8f = __half22float2(ds8[i]);
-
- sumf_d += ds8f.x * (sc[i] * sumi_d);
- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
- }
-
- const float2 dm4f = __half22float2(dm4);
-
- return dm4f.x * sumf_d - dm4f.y * sumf_m;
-#endif
-}
-
-#define VDR_Q5_K_Q8_1_MMVQ 2
-#define VDR_Q5_K_Q8_1_MMQ 8
-
-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
- const int* __restrict__ vl,
- const int* __restrict__ vh,
- const int* __restrict__ u,
- const uint8_t* __restrict__ sc,
- const uint8_t* __restrict__ m,
- const half2& dm5,
- const float* __restrict__ d8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
-
- float sumf_d = 0.0f;
- float sumf_m = 0.0f;
-
-#pragma unroll
- for (int i = 0; i < QR5_K; ++i) {
- const int vl0i = (vl[0] >> (4 * i)) & 0x0F0F0F0F;
- const int vl1i = (vl[1] >> (4 * i)) & 0x0F0F0F0F;
-
- const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
- const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
-
- const int v0i = vl0i | vh0i;
- const int v1i = vl1i | vh1i;
-
- const int dot1 = __dp4a(v0i, u[2 * i + 0], __dp4a(v1i, u[2 * i + 1], 0)); // SIMD dot product
- const int dot2 = __dp4a(0x01010101, u[2 * i + 0], __dp4a(0x01010101, u[2 * i + 1], 0)); // sum of u
-
- sumf_d += d8[i] * (dot1 * sc[i]);
- sumf_m += d8[i] * (dot2 * m[i]);
- }
-
- const float2 dm5f = __half22float2(dm5);
- return dm5f.x * sumf_d - dm5f.y * sumf_m;
-#endif
-}
-
-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
- const int* __restrict__ v,
- const int* __restrict__ u,
- const uint8_t* __restrict__ sc,
- const uint8_t* __restrict__ m,
- const half2& dm4,
- const half2* __restrict__ ds8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- float sumf_d = 0.0f;
- float sumf_m = 0.0f;
-
-#pragma unroll
- for (int i = 0; i < QR5_K * VDR_Q5_K_Q8_1_MMQ / QI8_1; ++i) {
- int sumi_d = 0;
-
-#pragma unroll
- for (int j = 0; j < QI8_1; ++j) {
- sumi_d = __dp4a(v[i * QI8_1 + j], u[i * QI8_1 + j], sumi_d); // SIMD dot product
- }
-
- const float2 ds8f = __half22float2(ds8[i]);
-
- sumf_d += ds8f.x * (sc[i] * sumi_d);
- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
- }
-
- const float2 dm4f = __half22float2(dm4);
-
- return dm4f.x * sumf_d - dm4f.y * sumf_m;
-#endif
-}
-
-#define VDR_Q6_K_Q8_1_MMVQ 1
-#define VDR_Q6_K_Q8_1_MMQ 8
-
-// contiguous v/x values
-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
- const int& vl,
- const int& vh,
- const int* __restrict__ u,
- const int8_t* __restrict__ scales,
- const float& d,
- const float* __restrict__ d8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- float sumf = 0.0f;
-
-#pragma unroll
- for (int i = 0; i < QR6_K; ++i) {
- const int sc = scales[4 * i];
- const int vil = (vl >> (4 * i)) & 0x0F0F0F0F;
- const int vih = ((vh >> (4 * i)) << 4) & 0x30303030;
- const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
-
- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product
- }
-
- return d * sumf;
-#endif
-}
-
-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
- const int* __restrict__ v,
- const int* __restrict__ u,
- const int8_t* __restrict__ sc,
- const float& d6,
- const float* __restrict__ d8) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- float sumf_d = 0.0f;
-
-#pragma unroll
- for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {
- int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale
-
-#pragma unroll
- for (int i = i0; i < i0 + 2; ++i) {
- sumi_d.x = __dp4a(v[2 * i + 0], u[2 * i + 0], sumi_d.x); // SIMD dot product
- sumi_d.x = __dp4a(v[2 * i + 1], u[2 * i + 1], sumi_d.x); // SIMD dot product
-
- sumi_d.y = __dp4a(v[2 * i + 4], u[2 * i + 4], sumi_d.y); // SIMD dot product
- sumi_d.y = __dp4a(v[2 * i + 5], u[2 * i + 5], sumi_d.y); // SIMD dot product
- }
-
- sumf_d += d8[i0 / 4] * (sc[i0 / 2 + 0] * sumi_d.x + sc[i0 / 2 + 1] * sumi_d.y);
- }
-
- return d6 * sumf_d;
-#endif
-}
-
-static __device__ __forceinline__ float
-vec_dot_q4_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q4_0* bq4_0 = (const block_q4_0*)vbq;
-
- int v[VDR_Q4_0_Q8_1_MMVQ];
- int u[2 * VDR_Q4_0_Q8_1_MMVQ];
-
-#pragma unroll
- for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
- v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);
- u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
- u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);
- }
-
- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, __half2float(bq4_0->d), bq8_1->ds);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q4_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI4_0) + mmq_y / QI4_0];
- *x_ql = tile_x_qs;
- *x_dm = (half2*)tile_x_d;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q4_0(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI4_0;
- const int kqsx = k % QI4_0;
-
- const block_q4_0* bx0 = (const block_q4_0*)vx;
- float* x_dmf = (float*)x_dm;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q4_0* bxi = bx0 + i * blocks_per_row + kbx;
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
- // x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbx] = bxi->d;
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_0;
- const int kbxd = k % blocks_per_tile_x_row;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) {
- int i = i0 + i_offset * QI4_0 + k / blocks_per_tile_x_row;
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q4_0* bxi = bx0 + i * blocks_per_row + kbxd;
- x_dmf[i * (WARP_SIZE_GGUF / QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- (void)x_qh;
- (void)x_sc;
-
- const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2));
- const float* x_dmf = (const float*)x_dm;
-
- int u[2 * VDR_Q4_0_Q8_1_MMQ];
-
-#pragma unroll
- for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {
- u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
- u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_0) % WARP_SIZE_GGUF];
- }
-
- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>(
- &x_ql[i * (WARP_SIZE_GGUF + 1) + k],
- u,
- x_dmf[i * (WARP_SIZE_GGUF / QI4_0) + i / QI4_0 + k / QI4_0],
- y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q4_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q4_1* bq4_1 = (const block_q4_1*)vbq;
-
- int v[VDR_Q4_1_Q8_1_MMVQ];
- int u[2 * VDR_Q4_1_Q8_1_MMVQ];
-
-#pragma unroll
- for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
- v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);
- u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
- u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);
- }
-
- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q4_1(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + +mmq_y];
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI4_1) + mmq_y / QI4_1];
- *x_ql = tile_x_qs;
- *x_dm = tile_x_dm;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q4_1(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI4_1;
- const int kqsx = k % QI4_1;
-
- const block_q4_1* bx0 = (const block_q4_1*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q4_1* bxi = bx0 + i * blocks_per_row + kbx;
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_1;
- const int kbxd = k % blocks_per_tile_x_row;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) {
- int i = i0 + i_offset * QI4_1 + k / blocks_per_tile_x_row;
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q4_1* bxi = bx0 + i * blocks_per_row + kbxd;
- x_dm[i * (WARP_SIZE_GGUF / QI4_1) + i / QI4_1 + kbxd] = bxi->dm;
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2));
-
- int u[2 * VDR_Q4_1_Q8_1_MMQ];
-
-#pragma unroll
- for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {
- u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
- u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_1) % WARP_SIZE_GGUF];
- }
-
- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>(
- &x_ql[i * (WARP_SIZE_GGUF + 1) + k],
- u,
- x_dm[i * (WARP_SIZE_GGUF / QI4_1) + i / QI4_1 + k / QI4_1],
- y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q5_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q5_0* bq5_0 = (const block_q5_0*)vbq;
-
- int vl[VDR_Q5_0_Q8_1_MMVQ];
- int vh[VDR_Q5_0_Q8_1_MMVQ];
- int u[2 * VDR_Q5_0_Q8_1_MMVQ];
-
-#pragma unroll
- for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
- vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);
- vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));
- u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
- u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);
- }
-
- return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, __half2float(bq5_0->d), bq8_1->ds);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q5_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y];
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI5_0) + mmq_y / QI5_0];
-
- *x_ql = tile_x_ql;
- *x_dm = (half2*)tile_x_d;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q5_0(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI5_0;
- const int kqsx = k % QI5_0;
-
- const block_q5_0* bx0 = (const block_q5_0*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
-
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q5_0* bxi = bx0 + i * blocks_per_row + kbx;
- const int ql = get_int_from_uint8(bxi->qs, kqsx);
- const int qh = get_int_from_uint8(bxi->qh, 0) >> (4 * (k % QI5_0));
-
- int qs0 = (ql >> 0) & 0x0F0F0F0F;
- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
- qs0 = __vsubss4(qs0, 0x10101010); // subtract 16
-
- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 0] = qs0;
-
- int qs1 = (ql >> 4) & 0x0F0F0F0F;
- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
- qs1 = __vsubss4(qs1, 0x10101010); // subtract 16
-
- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 1] = qs1;
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_0;
- const int kbxd = k % blocks_per_tile_x_row;
- float* x_dmf = (float*)x_dm;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) {
- int i = i0 + i_offset * QI5_0 + k / blocks_per_tile_x_row;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q5_0* bxi = bx0 + i * blocks_per_row + kbxd;
- x_dmf[i * (WARP_SIZE_GGUF / QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2));
- const int index_bx = i * (WARP_SIZE_GGUF / QI5_0) + i / QI5_0 + k / QI5_0;
- const float* x_dmf = (const float*)x_dm;
- const float* y_df = (const float*)y_ds;
-
- int u[2 * VDR_Q5_0_Q8_1_MMQ];
-
-#pragma unroll
- for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {
- u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
- u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_0) % WARP_SIZE_GGUF];
- }
-
- return vec_dot_q8_0_q8_1_impl<QR5_0 * VDR_Q5_0_Q8_1_MMQ>(
- &x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k],
- u,
- x_dmf[index_bx],
- y_df[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q5_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q5_1* bq5_1 = (const block_q5_1*)vbq;
-
- int vl[VDR_Q5_1_Q8_1_MMVQ];
- int vh[VDR_Q5_1_Q8_1_MMVQ];
- int u[2 * VDR_Q5_1_Q8_1_MMVQ];
-
-#pragma unroll
- for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
- vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);
- vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));
- u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
- u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);
- }
-
- return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q5_1(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y];
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI5_1) + mmq_y / QI5_1];
-
- *x_ql = tile_x_ql;
- *x_dm = tile_x_dm;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q5_1(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI5_1;
- const int kqsx = k % QI5_1;
-
- const block_q5_1* bx0 = (const block_q5_1*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q5_1* bxi = bx0 + i * blocks_per_row + kbx;
-
- const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);
- const int qh = get_int_from_uint8_aligned(bxi->qh, 0) >> (4 * (k % QI5_1));
-
- int qs0 = (ql >> 0) & 0x0F0F0F0F;
- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4
- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12
- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20
- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28
-
- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 0] = qs0;
-
- int qs1 = (ql >> 4) & 0x0F0F0F0F;
- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4
- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12
- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20
- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28
-
- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 1] = qs1;
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_1;
- const int kbxd = k % blocks_per_tile_x_row;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) {
- int i = i0 + i_offset * QI5_1 + k / blocks_per_tile_x_row;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q5_1* bxi = bx0 + i * blocks_per_row + kbxd;
-
- x_dm[i * (WARP_SIZE_GGUF / QI5_1) + i / QI5_1 + kbxd] = bxi->dm;
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2));
- const int index_bx = i * (WARP_SIZE_GGUF / QI5_1) + +i / QI5_1 + k / QI5_1;
-
- int u[2 * VDR_Q5_1_Q8_1_MMQ];
-
-#pragma unroll
- for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {
- u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];
- u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_1) % WARP_SIZE_GGUF];
- }
-
- return vec_dot_q8_1_q8_1_impl<QR5_1 * VDR_Q5_1_Q8_1_MMQ>(
- &x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k],
- u,
- x_dm[index_bx],
- y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q8_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q8_0* bq8_0 = (const block_q8_0*)vbq;
-
- int v[VDR_Q8_0_Q8_1_MMVQ];
- int u[VDR_Q8_0_Q8_1_MMVQ];
-
-#pragma unroll
- for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
- v[i] = get_int_from_int8(bq8_0->qs, iqs + i);
- u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
- }
-
- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, __half2float(bq8_0->d), __low2float(bq8_1->ds));
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q8_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI8_0) + mmq_y / QI8_0];
-
- *x_ql = tile_x_qs;
- *x_dm = (half2*)tile_x_d;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q8_0(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI8_0;
- const int kqsx = k % QI8_0;
- float* x_dmf = (float*)x_dm;
-
- const block_q8_0* bx0 = (const block_q8_0*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
-
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q8_0* bxi = bx0 + i * blocks_per_row + kbx;
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_int8(bxi->qs, kqsx);
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI8_0;
- const int kbxd = k % blocks_per_tile_x_row;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0) {
- int i = i0 + i_offset * QI8_0 + k / blocks_per_tile_x_row;
-
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q8_0* bxi = bx0 + i * blocks_per_row + kbxd;
- x_dmf[i * (WARP_SIZE_GGUF / QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- const float* x_dmf = (const float*)x_dm;
- const float* y_df = (const float*)y_ds;
-
- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>(
- &x_ql[i * (WARP_SIZE_GGUF + 1) + k],
- &y_qs[j * WARP_SIZE_GGUF + k],
- x_dmf[i * (WARP_SIZE_GGUF / QI8_0) + i / QI8_0 + k / QI8_0],
- y_df[j * (WARP_SIZE_GGUF / QI8_1) + k / QI8_1]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q2_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q2_K* bq2_K = (const block_q2_K*)vbq;
-
- const int bq8_offset = QR2_K * (iqs / QI8_1);
- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2);
-
- const uint8_t* scales = bq2_K->scales + scale_offset;
-
- const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);
- int u[QR2_K];
- float d8[QR2_K];
-
-#pragma unroll
- for (int i = 0; i < QR2_K; ++i) {
- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
- d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
- }
-
- return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q2_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI2_K) + mmq_y / QI2_K];
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 4) + mmq_y / 4];
-
- *x_ql = tile_x_ql;
- *x_dm = tile_x_dm;
- *x_sc = tile_x_sc;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q2_K(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI2_K;
- const int kqsx = k % QI2_K;
-
- const block_q2_K* bx0 = (const block_q2_K*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
-
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q2_K* bxi = bx0 + i * blocks_per_row + kbx;
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI2_K;
- const int kbxd = k % blocks_per_tile_x_row;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI2_K) {
- int i = (i0 + i_offset * QI2_K + k / blocks_per_tile_x_row) % mmq_y;
-
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q2_K* bxi = bx0 + i * blocks_per_row + kbxd;
- x_dm[i * (WARP_SIZE_GGUF / QI2_K) + i / QI2_K + kbxd] = bxi->dm;
- }
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
- int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF / 4);
-
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q2_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 4)) / (QI2_K / 4);
- x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + k % (WARP_SIZE_GGUF / 4)] =
- get_int_from_uint8_aligned(bxi->scales, k % (QI2_K / 4));
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- const int kbx = k / QI2_K;
- const int ky = (k % QI2_K) * QR2_K;
- const float* y_df = (const float*)y_ds;
-
- int v[QR2_K * VDR_Q2_K_Q8_1_MMQ];
-
- const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx * QI2_K + (QI2_K / 2) * (ky / (2 * QI2_K)) + ky % (QI2_K / 2);
- const int shift = 2 * ((ky % (2 * QI2_K)) / (QI2_K / 2));
-
-#pragma unroll
- for (int l = 0; l < QR2_K * VDR_Q2_K_Q8_1_MMQ; ++l) {
- v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;
- }
-
- const uint8_t* scales = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + kbx * 4]) + ky / 4;
-
- const int index_y = j * WARP_SIZE_GGUF + (QR2_K * k) % WARP_SIZE_GGUF;
- return vec_dot_q2_K_q8_1_impl_mmq(
- v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE_GGUF / QI2_K) + i / QI2_K + kbx], y_df[index_y / QI8_1]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q3_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q3_K* bq3_K = (const block_q3_K*)vbq;
-
- const int bq8_offset = QR3_K * (iqs / (QI3_K / 2));
- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2);
-
- const float d = __half2float(bq3_K->d);
-
- const int vl = get_int_from_uint8(bq3_K->qs, iqs);
-
- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
- const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K / 2)) >> bq8_offset;
-
- int u[QR3_K];
- float d8[QR3_K];
-
-#pragma unroll
- for (int i = 0; i < QR3_K; ++i) {
- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
- d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
- }
-
- return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q3_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI3_K) + mmq_y / QI3_K];
- __shared__ int tile_x_qh[mmq_y * (WARP_SIZE_GGUF / 2) + mmq_y / 2];
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 4) + mmq_y / 4];
-
- *x_ql = tile_x_ql;
- *x_dm = tile_x_dm;
- *x_qh = tile_x_qh;
- *x_sc = tile_x_sc;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q3_K(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI3_K;
- const int kqsx = k % QI3_K;
-
- const block_q3_K* bx0 = (const block_q3_K*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q3_K* bxi = bx0 + i * blocks_per_row + kbx;
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI3_K;
- const int kbxd = k % blocks_per_tile_x_row;
- float* x_dmf = (float*)x_dm;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI3_K) {
- int i = (i0 + i_offset * QI3_K + k / blocks_per_tile_x_row) % mmq_y;
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q3_K* bxi = bx0 + i * blocks_per_row + kbxd;
- x_dmf[i * (WARP_SIZE_GGUF / QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);
- }
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {
- int i = i0 + i_offset * 2 + k / (WARP_SIZE_GGUF / 2);
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q3_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 2)) / (QI3_K / 2);
- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
- x_qh[i * (WARP_SIZE_GGUF / 2) + i / 2 + k % (WARP_SIZE_GGUF / 2)] =
- ~get_int_from_uint8(bxi->hmask, k % (QI3_K / 2));
- }
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {
- int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF / 4);
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q3_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 4)) / (QI3_K / 4);
-
- const int ksc = k % (QI3_K / 4);
-
- const int ksc_low = ksc % (QI3_K / 8);
- const int shift_low = 4 * (ksc / (QI3_K / 8));
- const int sc_low = (get_int_from_uint8(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;
-
- const int ksc_high = QI3_K / 8;
- const int shift_high = 2 * ksc;
- const int sc_high = ((get_int_from_uint8(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;
-
- const int sc = __vsubss4(sc_low | sc_high, 0x20202020);
-
- x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + k % (WARP_SIZE_GGUF / 4)] = sc;
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- const int kbx = k / QI3_K;
- const int ky = (k % QI3_K) * QR3_K;
- const float* x_dmf = (const float*)x_dm;
- const float* y_df = (const float*)y_ds;
-
- const int8_t* scales = ((const int8_t*)(x_sc + i * (WARP_SIZE_GGUF / 4) + i / 4 + kbx * 4)) + ky / 4;
-
- int v[QR3_K * VDR_Q3_K_Q8_1_MMQ];
-
-#pragma unroll
- for (int l = 0; l < QR3_K * VDR_Q3_K_Q8_1_MMQ; ++l) {
- const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx * QI3_K + (QI3_K / 2) * (ky / (2 * QI3_K)) + ky % (QI3_K / 2);
- const int shift = 2 * ((ky % 32) / 8);
- const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;
-
- const int vh = x_qh[i * (WARP_SIZE_GGUF / 2) + i / 2 + kbx * (QI3_K / 2) + (ky + l) % 8] >> ((ky + l) / 8);
- const int vlh = (vh << 2) & 0x04040404;
-
- v[l] = __vsubss4(vll, vlh);
- }
-
- const int index_y = j * WARP_SIZE_GGUF + (k * QR3_K) % WARP_SIZE_GGUF;
- return vec_dot_q3_K_q8_1_impl_mmq(
- v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE_GGUF / QI3_K) + i / QI3_K + kbx], y_df[index_y / QI8_1]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q4_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q4_K* bq4_K = (const block_q4_K*)vbq;
-
- int v[2];
- int u[2 * QR4_K];
- float d8[QR4_K];
-
- // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
- const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));
-
- // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
- // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
- // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
- // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108
-
- const int* q4 = (const int*)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));
- v[0] = q4[0];
- v[1] = q4[4];
-
- const uint16_t* scales = (const uint16_t*)bq4_K->scales;
- uint16_t aux[2];
- const int j = bq8_offset / 2;
- if (j < 2) {
- aux[0] = scales[j + 0] & 0x3f3f;
- aux[1] = scales[j + 2] & 0x3f3f;
- } else {
- aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);
- aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);
- }
- const uint8_t* sc = (const uint8_t*)aux;
- const uint8_t* m = sc + 2;
-
- for (int i = 0; i < QR4_K; ++i) {
- const block_q8_1* bq8i = bq8_1 + bq8_offset + i;
- d8[i] = __low2float(bq8i->ds);
-
- const int* q8 = (const int*)bq8i->qs + ((iqs / 2) % 4);
- u[2 * i + 0] = q8[0];
- u[2 * i + 1] = q8[4];
- }
-
- return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q4_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI4_K) + mmq_y / QI4_K];
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8];
-
- *x_ql = tile_x_ql;
- *x_dm = tile_x_dm;
- *x_sc = tile_x_sc;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q4_K(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI4_K; // == 0 if QK_K == 256
- const int kqsx = k % QI4_K; // == k if QK_K == 256
-
- const block_q4_K* bx0 = (const block_q4_K*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
-
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q4_K* bxi = bx0 + i * blocks_per_row + kbx;
- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_K; // == 1 if QK_K == 256
- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_K) {
- int i = (i0 + i_offset * QI4_K + k / blocks_per_tile_x_row) % mmq_y;
- if (need_check) {
- i = min(i, i_max);
- }
- const block_q4_K* bxi = bx0 + i * blocks_per_row + kbxd;
- x_dm[i * (WARP_SIZE_GGUF / QI4_K) + i / QI4_K + kbxd] = bxi->dm;
- }
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q4_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / (QI4_K / 8);
-
- const int* scales = (const int*)bxi->scales;
-
- const int ksc = k % (WARP_SIZE_GGUF / 8);
- // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
- int scales8 = (scales[(ksc % 2) + (ksc != 0)] >> (4 * (ksc & (ksc / 2)))) & 0x0F0F0F0F; // lower 4 bits
- scales8 |= (scales[ksc / 2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
-
- x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + ksc] = scales8;
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- (void)x_qh;
-
- const uint8_t* sc = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 16]) + 2 * ((k % 16) / 8);
-
- const int index_y = j * WARP_SIZE_GGUF + (QR4_K * k) % WARP_SIZE_GGUF;
- return vec_dot_q4_K_q8_1_impl_mmq(
- &x_ql[i * (WARP_SIZE_GGUF + 1) + k],
- &y_qs[index_y],
- sc,
- sc + 8,
- x_dm[i * (WARP_SIZE_GGUF / QI4_K) + i / QI4_K],
- &y_ds[index_y / QI8_1]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q5_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q5_K* bq5_K = (const block_q5_K*)vbq;
-
- int vl[2];
- int vh[2];
- int u[2 * QR5_K];
- float d8[QR5_K];
-
- const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2));
- const int* ql = (const int*)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));
- const int* qh = (const int*)(bq5_K->qh + 4 * ((iqs / 2) % 4));
-
- vl[0] = ql[0];
- vl[1] = ql[4];
-
- vh[0] = qh[0] >> bq8_offset;
- vh[1] = qh[4] >> bq8_offset;
-
- const uint16_t* scales = (const uint16_t*)bq5_K->scales;
- uint16_t aux[2];
- const int j = bq8_offset / 2;
- if (j < 2) {
- aux[0] = scales[j + 0] & 0x3f3f;
- aux[1] = scales[j + 2] & 0x3f3f;
- } else {
- aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);
- aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);
- }
- const uint8_t* sc = (const uint8_t*)aux;
- const uint8_t* m = sc + 2;
-
-#pragma unroll
- for (int i = 0; i < QR5_K; ++i) {
- const block_q8_1* bq8i = bq8_1 + bq8_offset + i;
- d8[i] = __low2float(bq8i->ds);
-
- const int* q8 = (const int*)bq8i->qs + ((iqs / 2) % 4);
- u[2 * i + 0] = q8[0];
- u[2 * i + 1] = q8[4];
- }
-
- return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q5_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y];
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI5_K) + mmq_y / QI5_K];
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8];
-
- *x_ql = tile_x_ql;
- *x_dm = tile_x_dm;
- *x_sc = tile_x_sc;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q5_K(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI5_K; // == 0 if QK_K == 256
- const int kqsx = k % QI5_K; // == k if QK_K == 256
-
- const block_q5_K* bx0 = (const block_q5_K*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q5_K* bxi = bx0 + i * blocks_per_row + kbx;
- const int ky = QR5_K * kqsx;
-
- const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);
- const int ql0 = (ql >> 0) & 0x0F0F0F0F;
- const int ql1 = (ql >> 4) & 0x0F0F0F0F;
-
- const int qh = get_int_from_uint8_aligned(bxi->qh, kqsx % (QI5_K / 4));
- const int qh0 = ((qh >> (2 * (kqsx / (QI5_K / 4)) + 0)) << 4) & 0x10101010;
- const int qh1 = ((qh >> (2 * (kqsx / (QI5_K / 4)) + 1)) << 4) & 0x10101010;
-
- const int kq0 = ky - ky % (QI5_K / 2) + k % (QI5_K / 4) + 0;
- const int kq1 = ky - ky % (QI5_K / 2) + k % (QI5_K / 4) + (QI5_K / 4);
-
- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq0] = ql0 | qh0;
- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq1] = ql1 | qh1;
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_K; // == 1 if QK_K == 256
- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_K) {
- int i = (i0 + i_offset * QI5_K + k / blocks_per_tile_x_row) % mmq_y;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q5_K* bxi = bx0 + i * blocks_per_row + kbxd;
- x_dm[i * (WARP_SIZE_GGUF / QI5_K) + i / QI5_K + kbxd] = bxi->dm;
- }
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q5_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / (QI5_K / 8);
-
- const int* scales = (const int*)bxi->scales;
-
- const int ksc = k % (WARP_SIZE_GGUF / 8);
-
- // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8
- int scales8 = (scales[(ksc % 2) + (ksc != 0)] >> (4 * (ksc & (ksc / 2)))) & 0x0F0F0F0F; // lower 4 bits
- scales8 |= (scales[ksc / 2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits
-
- x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + ksc] = scales8;
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- const uint8_t* sc = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 16]) + 2 * ((k % 16) / 8);
-
- const int index_x = i * (QR5_K * WARP_SIZE_GGUF + 1) + QR5_K * k;
- const int index_y = j * WARP_SIZE_GGUF + (QR5_K * k) % WARP_SIZE_GGUF;
- return vec_dot_q5_K_q8_1_impl_mmq(
- &x_ql[index_x],
- &y_qs[index_y],
- sc,
- sc + 8,
- x_dm[i * (WARP_SIZE_GGUF / QI5_K) + i / QI5_K],
- &y_ds[index_y / QI8_1]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_q6_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_q6_K* bq6_K = (const block_q6_K*)vbq;
-
- const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 4);
- const int scale_offset = (QI6_K / 4) * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 8);
- const int vh_shift = 2 * ((iqs % (QI6_K / 2)) / (QI6_K / 4));
-
- const int vl = get_int_from_uint8(bq6_K->ql, iqs);
- const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K / 4) * (iqs / (QI6_K / 2)) + iqs % (QI6_K / 4)) >> vh_shift;
-
- const int8_t* scales = bq6_K->scales + scale_offset;
-
- int u[QR6_K];
- float d8[QR6_K];
-
-#pragma unroll
- for (int i = 0; i < QR6_K; ++i) {
- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2 * i].qs, iqs % QI8_1);
- d8[i] = __low2float(bq8_1[bq8_offset + 2 * i].ds);
- }
-
- return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, __half2float(bq6_K->d), d8);
-}
-
-template <int mmq_y>
-static __device__ __forceinline__ void allocate_tiles_q6_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {
- __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y];
- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI6_K) + mmq_y / QI6_K];
- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8];
-
- *x_ql = tile_x_ql;
- *x_dm = tile_x_dm;
- *x_sc = tile_x_sc;
-}
-
-template <int mmq_y, int nwarps, bool need_check>
-static __device__ __forceinline__ void load_tiles_q6_K(
- const void* __restrict__ vx,
- int* __restrict__ x_ql,
- half2* __restrict__ x_dm,
- int* __restrict__ x_qh,
- int* __restrict__ x_sc,
- const int& i_offset,
- const int& i_max,
- const int& k,
- const int& blocks_per_row) {
- const int kbx = k / QI6_K; // == 0 if QK_K == 256
- const int kqsx = k % QI6_K; // == k if QK_K == 256
-
- const block_q6_K* bx0 = (const block_q6_K*)vx;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {
- int i = i0 + i_offset;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q6_K* bxi = bx0 + i * blocks_per_row + kbx;
- const int ky = QR6_K * kqsx;
-
- const int ql = get_int_from_uint8(bxi->ql, kqsx);
- const int ql0 = (ql >> 0) & 0x0F0F0F0F;
- const int ql1 = (ql >> 4) & 0x0F0F0F0F;
-
- const int qh = get_int_from_uint8(bxi->qh, (QI6_K / 4) * (kqsx / (QI6_K / 2)) + kqsx % (QI6_K / 4));
- const int qh0 = ((qh >> (2 * ((kqsx % (QI6_K / 2)) / (QI6_K / 4)))) << 4) & 0x30303030;
- const int qh1 = (qh >> (2 * ((kqsx % (QI6_K / 2)) / (QI6_K / 4)))) & 0x30303030;
-
- const int kq0 = ky - ky % QI6_K + k % (QI6_K / 2) + 0;
- const int kq1 = ky - ky % QI6_K + k % (QI6_K / 2) + (QI6_K / 2);
-
- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);
- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);
- }
-
- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI6_K; // == 1 if QK_K == 256
- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256
- float* x_dmf = (float*)x_dm;
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) {
- int i = (i0 + i_offset * QI6_K + k / blocks_per_tile_x_row) % mmq_y;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q6_K* bxi = bx0 + i * blocks_per_row + kbxd;
-
- x_dmf[i * (WARP_SIZE_GGUF / QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);
- }
-
-#pragma unroll
- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {
- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y;
-
- if (need_check) {
- i = min(i, i_max);
- }
-
- const block_q6_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / 4;
-
- x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k % (WARP_SIZE_GGUF / 8)] = get_int_from_int8(bxi->scales, k % (QI6_K / 8));
- }
-}
-
-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(
- const int* __restrict__ x_ql,
- const half2* __restrict__ x_dm,
- const int* __restrict__ x_qh,
- const int* __restrict__ x_sc,
- const int* __restrict__ y_qs,
- const half2* __restrict__ y_ds,
- const int& i,
- const int& j,
- const int& k) {
- const float* x_dmf = (const float*)x_dm;
- const float* y_df = (const float*)y_ds;
-
- const int8_t* sc = ((const int8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 8]);
-
- const int index_x = i * (QR6_K * WARP_SIZE_GGUF + 1) + QR6_K * k;
- const int index_y = j * WARP_SIZE_GGUF + (QR6_K * k) % WARP_SIZE_GGUF;
- return vec_dot_q6_K_q8_1_impl_mmq(
- &x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE_GGUF / QI6_K) + i / QI6_K], &y_df[index_y / QI8_1]);
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq2_xxs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_iq2_xxs* bq2 = (const block_iq2_xxs*)vbq;
-
- const int ib32 = iqs;
- const uint16_t* q2 = bq2->qs + 4 * ib32;
- const uint8_t* aux8 = (const uint8_t*)q2;
- const int8_t* q8 = bq8_1[ib32].qs;
- uint32_t aux32 = q2[2] | (q2[3] << 16);
- int sumi = 0;
- for (int l = 0; l < 4; ++l) {
- const uint8_t* grid = (const uint8_t*)(iq2xxs_grid + aux8[l]);
- const uint8_t signs = ksigns_iq2xs[aux32 & 127];
- for (int j = 0; j < 8; ++j) {
- sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
- }
- q8 += 8;
- aux32 >>= 7;
- }
- const float d = __half2float(bq2->d) * (0.5f + aux32) * __half2float(bq8_1[ib32].ds.x) * 0.25f;
- return d * sumi;
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq2_xs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
- const block_iq2_xs* bq2 = (const block_iq2_xs*)vbq;
-
- const int ib32 = iqs;
- const uint16_t* q2 = bq2->qs + 4 * ib32;
- const int8_t* q8 = bq8_1[ib32].qs;
- const uint8_t ls1 = bq2->scales[ib32] & 0xf;
- const uint8_t ls2 = bq2->scales[ib32] >> 4;
- int sumi1 = 0;
- for (int l = 0; l < 2; ++l) {
- const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[l] & 511));
- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
- for (int j = 0; j < 8; ++j) {
- sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
- }
- q8 += 8;
- }
- int sumi2 = 0;
- for (int l = 2; l < 4; ++l) {
- const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[l] & 511));
- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
- for (int j = 0; j < 8; ++j) {
- sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
- }
- q8 += 8;
- }
- const float d = __half2float(bq2->d) * __half2float(bq8_1[ib32].ds.x) * 0.25f;
- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq2_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- const block_iq2_s* bq2 = (const block_iq2_s*)vbq;
-
- const int ib32 = iqs;
- const int8_t* q8 = bq8_1[ib32].qs;
- const uint8_t* signs = bq2->qs + QK_K / 8 + 4 * ib32;
- const uint8_t ls1 = bq2->scales[ib32] & 0xf;
- const uint8_t ls2 = bq2->scales[ib32] >> 4;
- int sumi1 = 0;
- for (int l = 0; l < 2; ++l) {
- const uint32_t* grid =
- (const uint32_t*)(iq2s_grid + (bq2->qs[4 * ib32 + l] | ((bq2->qh[ib32] << (8 - 2 * l)) & 0x300)));
- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);
- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);
- sumi1 = __dp4a(grid_l, *((const int*)q8 + 0), sumi1);
- sumi1 = __dp4a(grid_h, *((const int*)q8 + 1), sumi1);
- q8 += 8;
- }
- int sumi2 = 0;
- for (int l = 2; l < 4; ++l) {
- const uint32_t* grid =
- (const uint32_t*)(iq2s_grid + (bq2->qs[4 * ib32 + l] | ((bq2->qh[ib32] << (8 - 2 * l)) & 0x300)));
- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);
- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);
- sumi2 = __dp4a(grid_l, *((const int*)q8 + 0), sumi2);
- sumi2 = __dp4a(grid_h, *((const int*)q8 + 1), sumi2);
- q8 += 8;
- }
- const float d = __half2float(bq2->d) * __low2float(bq8_1[ib32].ds) * 0.25f;
- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
-#endif
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq3_xxs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- const block_iq3_xxs* bq2 = (const block_iq3_xxs*)vbq;
-
- const int ib32 = iqs;
- const uint8_t* q3 = bq2->qs + 8 * ib32;
- const uint16_t* gas = (const uint16_t*)(bq2->qs + QK_K / 4) + 2 * ib32;
- const int8_t* q8 = bq8_1[ib32].qs;
- uint32_t aux32 = gas[0] | (gas[1] << 16);
- int sumi = 0;
- for (int l = 0; l < 4; ++l) {
- const uint32_t* grid1 = iq3xxs_grid + q3[2 * l + 0];
- const uint32_t* grid2 = iq3xxs_grid + q3[2 * l + 1];
- const uint32_t* signs = (const uint32_t*)(ksigns64 + (aux32 & 127));
- const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);
- const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);
- sumi = __dp4a(grid_l, *((int*)q8 + 0), sumi);
- sumi = __dp4a(grid_h, *((int*)q8 + 1), sumi);
- q8 += 8;
- aux32 >>= 7;
- }
- const float d = __half2float(bq2->d) * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;
- return d * sumi;
-#endif
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq3_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- const block_iq3_s* bq2 = (const block_iq3_s*)vbq;
-
- const int ib32 = iqs;
- const uint8_t* qs = bq2->qs + 8 * ib32;
- const int8_t* q8 = bq8_1[ib32].qs;
- int sumi = 0;
- for (int l = 0; l < 4; ++l) {
- const uint32_t* grid1 = iq3xs_grid + (qs[2 * l + 0] | ((bq2->qh[ib32] << (8 - 2 * l)) & 256));
- const uint32_t* grid2 = iq3xs_grid + (qs[2 * l + 1] | ((bq2->qh[ib32] << (7 - 2 * l)) & 256));
- uint32_t signs0 = __vcmpeq4(((bq2->signs[4 * ib32 + l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);
- uint32_t signs1 = __vcmpeq4(((bq2->signs[4 * ib32 + l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);
- const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);
- const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);
- sumi = __dp4a(grid_l, *((int*)q8 + 0), sumi);
- sumi = __dp4a(grid_h, *((int*)q8 + 1), sumi);
- q8 += 8;
- }
- const float d = __half2float(bq2->d) * (0.5f + ((bq2->scales[ib32 / 2] >> 4 * (ib32 % 2)) & 0xf)) *
- __low2float(bq8_1[ib32].ds) * 0.5f;
- return d * sumi;
-#endif
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq1_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- const block_iq1_s* bq1 = (const block_iq1_s*)vbq;
-
- const int qs_packed = get_int_b2(bq1->qs, iqs);
- const uint8_t* qs = (const uint8_t*)&qs_packed;
-
- const int qh = bq1->qh[iqs];
-
- int sumi = 0;
-#pragma unroll
- for (int l0 = 0; l0 < 8; l0 += 2) {
- const int grid = iq1s_grid_gpu[qs[l0 / 2] | (((qh >> 3 * (l0 / 2)) & 0x07) << 8)];
-
- const int grid0 = (grid >> 0) & 0x0F0F0F0F;
- const int grid1 = (grid >> 4) & 0x0F0F0F0F;
-
- const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
- const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
-
- sumi = __dp4a(grid0, u0, sumi);
- sumi = __dp4a(grid1, u1, sumi);
- }
-
- const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);
- const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f * IQ1S_DELTA / 0x8000);
- const float2 ds = __half22float2(bq8_1[iqs].ds);
- return d1q * (ds.x * sumi + ds.y * delta);
-#endif
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq1_m_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
-
- const block_iq1_m* bq1 = (const block_iq1_m*)vbq;
-
- const int qs_packed = get_int_b4(bq1->qs, iqs);
- const uint8_t* qs = (const uint8_t*)&qs_packed;
-
- int sumi[2] = {0};
- float sumf[2] = {0.0f};
-#pragma unroll
- for (int l0 = 0; l0 < 8; l0 += 2) {
- const int qhl = bq1->qh[2 * iqs + l0 / 4] >> (4 * ((l0 / 2) % 2));
-
- const int grid = iq1s_grid_gpu[qs[l0 / 2] | ((qhl & 0x07) << 8)];
-
- const int grid0 = (grid >> 0) & 0x0F0F0F0F;
- const int grid1 = (grid >> 4) & 0x0F0F0F0F;
-
- const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
- const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
-
- sumi[l0 / 4] = __dp4a(grid0, u0, sumi[l0 / 4]);
- sumi[l0 / 4] = __dp4a(grid1, u1, sumi[l0 / 4]);
-
- const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f * IQ1M_DELTA / 0x08);
- int sumy = 0;
- sumy = __dp4a(u0, 0x01010101, sumy);
- sumy = __dp4a(u1, 0x01010101, sumy);
- sumf[l0 / 4] += delta * sumy;
- }
-
- const uint16_t* sc = (const uint16_t*)bq1->scales;
-
- iq1m_scale_t scale;
- scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
- const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);
-
- const int tmp = sc[iqs / 2] >> (6 * (iqs % 2));
- const int sc0 = 2 * ((tmp >> 0) & 0x07) + 1;
- const int sc1 = 2 * ((tmp >> 3) & 0x07) + 1;
- return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);
-#endif
-}
-
-static __device__ __forceinline__ void
-get_int_from_table_16(const uint32_t& q4, const uint8_t* values, int& val1, int& val2) {
- uint32_t aux32;
- const uint8_t* q8 = (const uint8_t*)&aux32;
- aux32 = q4 & 0x0f0f0f0f;
- uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);
- uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);
- val1 = v1 | (v2 << 16);
- aux32 = (q4 >> 4) & 0x0f0f0f0f;
- v1 = values[q8[0]] | (values[q8[1]] << 8);
- v2 = values[q8[2]] | (values[q8[3]] << 8);
- val2 = v1 | (v2 << 16);
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq4_nl_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
-
- const block_iq4_nl* bq = (const block_iq4_nl*)vbq;
-
- const uint16_t* q4 = (const uint16_t*)bq->qs + 2 * iqs;
- const int32_t* q8 = (const int32_t*)bq8_1->qs + iqs;
-
- const uint8_t* values = (const uint8_t*)kvalues_iq4nl;
-
- int v1, v2;
- int sumi1 = 0, sumi2 = 0;
- for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
- const uint32_t aux = q4[2 * l] | (q4[2 * l + 1] << 16);
- get_int_from_table_16(aux, values, v1, v2);
- sumi1 = __dp4a(v1, q8[l + 0], sumi1);
- sumi2 = __dp4a(v2, q8[l + 4], sumi2);
- }
- const float d = __half2float(bq->d) * __low2float(bq8_1->ds);
- return d * (sumi1 + sumi2);
-#endif
-}
-
-static __device__ __forceinline__ float
-vec_dot_iq4_xs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA
- const block_iq4_xs* bq4 = (const block_iq4_xs*)vbq;
- const uint8_t* values = (const uint8_t*)kvalues_iq4nl;
-
- // iqs is 0...7
- const int ib32 = iqs;
- const int32_t* q8 = (const int*)bq8_1[ib32].qs;
- const uint32_t* q4 = (const uint32_t*)bq4->qs + 4 * ib32;
- const int8_t ls = ((bq4->scales_l[ib32 / 2] >> 4 * (ib32 % 2)) & 0xf) | (((bq4->scales_h >> 2 * ib32) & 3) << 4);
- const float d = __half2float(bq4->d) * (ls - 32) * __low2float(bq8_1[ib32].ds);
- int v1, v2;
- int sumi1 = 0, sumi2 = 0;
- for (int j = 0; j < 4; ++j) {
- get_int_from_table_16(q4[j], values, v1, v2);
- sumi1 = __dp4a(v1, q8[j + 0], sumi1);
- sumi2 = __dp4a(v2, q8[j + 4], sumi2);
- }
- return d * (sumi1 + sumi2);
-#endif
-}
diff --git a/python/freetoken/kernel/ggml.py b/python/freetoken/kernel/ggml.py
new file mode 100644
index 00000000..b672d58c
--- /dev/null
+++ b/python/freetoken/kernel/ggml.py
@@ -0,0 +1,155 @@
+"""llama.cpp's ggml-cuda kernels (vendored under ``csrc/ggml``) as a tvm-ffi module: prebuilt in the kernel cache, else compiled on first use.
+
+Weights stay in their packed ggml blocks. ``mul_mat`` / ``mul_mat_id`` pick MMVQ or MMQ the way llama.cpp's mul_mat / mul_mat_id do.
+Activations are fp32, bf16 or fp16 and are quantized to q8_1 first; results come out in the activation dtype and are accumulated in fp32.
+"""
+
+from __future__ import annotations
+
+import functools
+import pathlib
+from typing import TYPE_CHECKING
+
+import torch
+
+from .utils import _is_rocm, load_aot
+
+if TYPE_CHECKING:
+ from tvm_ffi import Module
+
+_ROOT = pathlib.Path(__file__).parent / "csrc" / "ggml"
+
+
+def load_module(build_directory: str | None = None) -> Module:
+ cuda_dir = _ROOT / "ggml-cuda"
+ sources = [
+ _ROOT / "freetoken" / "ggml_shim.cu",
+ _ROOT / "freetoken" / "ggml_ops.cu",
+ *(cuda_dir / f for f in ("mmvq.cu", "mmq.cu", "mmid.cu", "quantize.cu", "convert.cu", "getrows.cu")),
+ *sorted((cuda_dir / "template-instances").glob("mmq-instance-*.cu")),
+ ]
+ if _is_rocm():
+ flags = ["-ffast-math", "-DGGML_USE_HIP"]
+ else:
+ # a multi-arch kernel-cache build is ~6x smaller with compressed SASS
+ flags = ["-use_fast_math", "--extended-lambda", "--compress-mode=size"]
+ return load_aot(
+ "ggml",
+ cuda_files=[str(s) for s in sources],
+ extra_cuda_cflags=flags,
+ extra_include_paths=[str(_ROOT / "include"), str(_ROOT / "src"), str(cuda_dir)],
+ build_directory=build_directory,
+ )
+
+
+def _module() -> Module:
+ return load_module()
+
+
+def _mmq_tail_bytes(ggml_type: int, k: int) -> int:
+ return _module().mmq_tail_bytes(ggml_type, k)
+
+
+def _check_mmq_tail(w: torch.Tensor, ggml_type: int, k: int) -> None:
+ """MMQ tiles read up to MATRIX_ROW_PADDING values past the last row of ``w``, so its storage must hold them."""
+ tail = _mmq_tail_bytes(ggml_type, k)
+ if tail == 0 or w.numel() == 0:
+ return
+ end = w.storage_offset() + sum((n - 1) * s for n, s in zip(w.shape, w.stride())) + 1
+ if end + tail > w.untyped_storage().nbytes():
+ raise ValueError(f"ggml type {ggml_type} K={k} weight needs {tail} bytes of zero padding after its last row for MMQ")
+
+
+def mul_mat(w: torch.Tensor, ggml_type: int, x: torch.Tensor, *, out: torch.Tensor | None = None, xq: torch.Tensor | None = None) -> torch.Tensor:
+ """``x[M, K] @ w[N, K]^T`` in ``x``'s dtype with ``w`` packed as ``[N, row_bytes]`` uint8, or written into the ``[M, N]`` view ``out``.
+
+ ``xq`` is ``quantize_q8_1(x)``: the vector kernel reads it instead of quantizing ``x`` again.
+ """
+ m, k = x.shape
+ if out is None:
+ out = torch.empty(m, w.shape[0], dtype=x.dtype, device=x.device)
+ if not uses_mmvq(ggml_type, m):
+ _check_mmq_tail(w, ggml_type, k)
+ _module().mul_mat(w, ggml_type, x.contiguous(), out, xq)
+ return out
+
+
+def quantize_q8_1(x: torch.Tensor) -> torch.Tensor:
+ """``x[M, K]`` as the vector kernel's q8_1 blocks, to share among GEMVs that read the same ``x``."""
+ m, k = x.shape
+ out = torch.empty(m, _q8_1_row_bytes(k), dtype=torch.uint8, device=x.device)
+ _module().quantize_q8_1(x.contiguous(), out)
+ return out
+
+
+def _q8_1_row_bytes(k: int) -> int:
+ return _module().q8_1_row_bytes(k)
+
+
+def uses_mmvq(ggml_type: int, m: int) -> bool:
+ """Whether ``mul_mat`` of ``m`` rows on this GPU runs the vector kernel, the one that reads ``xq``."""
+ return _module().uses_mmvq(ggml_type, m)
+
+
+def mul_mat_id(w: torch.Tensor, ggml_type: int, x: torch.Tensor, ids: torch.Tensor) -> torch.Tensor:
+ """Per-route expert GEMM over ``w[E, N, row_bytes]``: returns ``[T, k, N]`` in ``x``'s dtype; ``x`` is ``[T, K]`` (shared by the k routes) or ``[T, k, K]``."""
+ t, top_k = ids.shape
+ out = torch.empty(t, top_k, w.shape[1], dtype=x.dtype, device=x.device)
+ if t > mmvq_max_tokens(ggml_type):
+ _check_mmq_tail(w, ggml_type, x.shape[-1])
+ _module().mul_mat_id(w, ggml_type, (x.unsqueeze(1) if x.dim() == 2 else x).contiguous(), ids, out)
+ return out
+
+
+def mul_mat_id_glu(w: torch.Tensor, ggml_type: int, x: torch.Tensor, ids: torch.Tensor, glu: str, limit: float = 0.0) -> torch.Tensor:
+ """``glu(gate, up)`` per route in one pass for small ``T``: ``w[E, 2I, row_bytes]`` holds gate rows then up rows; returns ``[T, k, I]`` in ``x``'s dtype.
+
+ ``glu`` is one of ``swiglu``, ``geglu`` (tanh GELU), ``swiglu_oai`` (alpha 1.702, limit 7) and ``swiglu_clamp`` (alpha 1, clamped at ``limit``).
+ """
+ t, top_k = ids.shape
+ out = torch.empty(t, top_k, w.shape[1] // 2, dtype=x.dtype, device=x.device)
+ _module().mul_mat_id_glu(w, ggml_type, x.contiguous(), ids, out, glu, limit)
+ return out
+
+
+def moe_weighted_sum(y: torch.Tensor, weights: torch.Tensor) -> torch.Tensor:
+ """``sum_j weights[t, j] * y[t, j, :]`` over ``y[T, k, N]``, accumulated in fp32 and returned in ``y``'s dtype."""
+ out = torch.empty(y.shape[0], y.shape[2], dtype=y.dtype, device=y.device)
+ _module().moe_weighted_sum(y.contiguous(), weights.float().contiguous(), out)
+ return out
+
+
+def mmvq_max_tokens(ggml_type: int) -> int:
+ """Largest token count the small-batch MoE kernels take on this GPU."""
+ return _module().mmvq_max_tokens(ggml_type)
+
+
+def dequantize(w: torch.Tensor, ggml_type: int, k: int, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
+ out = torch.empty(w.numel() // w.shape[-1], k, dtype=dtype, device=w.device)
+ _module().dequantize(w, ggml_type, k, out)
+ return out
+
+
+def get_rows(w: torch.Tensor, ggml_type: int, k: int, ids: torch.Tensor, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
+ out = torch.empty(ids.shape[0], k, dtype=dtype, device=w.device)
+ _module().get_rows(w, ggml_type, k, ids, out)
+ return out
+
+
+__all__ = [
+ "dequantize",
+ "get_rows",
+ "load_module",
+ "mmvq_max_tokens",
+ "moe_weighted_sum",
+ "mul_mat",
+ "mul_mat_id",
+ "mul_mat_id_glu",
+ "quantize_q8_1",
+ "uses_mmvq",
+]
diff --git a/python/freetoken/kernel/gguf.py b/python/freetoken/kernel/gguf.py
deleted file mode 100644
index 04a16560..00000000
--- a/python/freetoken/kernel/gguf.py
+++ /dev/null
@@ -1,142 +0,0 @@
-"""Borrowed llama.cpp GGUF dequant/GEMM CUDA kernels, JIT-compiled on first use.
-
-The ``.cu``/``.cuh`` under ``csrc/gguf/`` are vendored verbatim from sgl-kernel
-(``csrc/quantization/gguf/``), which are themselves ports of llama.cpp. We compile
-them through ``torch.utils.cpp_extension.load`` (the same toolchain sglang/vllm use)
-into a torch-op module and expose the handful of ops the GGUF path needs. This is a
-separate, torch-native extension that sits alongside FreeToken's tvm-ffi kernels.
-
-All ops keep the weight in its native GGUF block layout (packed ``uint8`` rows) and
-dequantize *inside* the kernel -- no bf16 copy of the weight is ever materialized.
-"""
-
-from __future__ import annotations
-
-import functools
-import os
-import pathlib
-import shutil
-
-import torch
-
-_CSRC = pathlib.Path(__file__).parent / "csrc" / "gguf"
-
-
-def _host_compiler() -> str | None:
- """A host compiler nvcc + libtorch headers accept.
-
- The system default gcc can be too new for the torch headers (gcc 16 hard-errors),
- and on this toolchain even nvcc+gcc-13 trips a non-conformant ``typename
- decltype`` in ``List_inl.h`` once ``torch::Tensor`` is instantiated -- but nvcc
- with ``clang++`` as host compiles it cleanly. So prefer clang++, then fall back
- to an older gcc. Override with ``FREETOKEN_GGUF_HOST_CXX``.
- """
- override = os.environ.get("FREETOKEN_GGUF_HOST_CXX")
- if override:
- return override
- for cxx in ("clang++", "g++-13", "g++-14", "g++-15"):
- if shutil.which(cxx):
- return cxx
- return None
-
-
-def _c_compiler_for(cxx: str) -> str:
- base = os.path.basename(cxx)
- if "clang" in base:
- return shutil.which("clang") or "clang"
- cc = base.replace("g++", "gcc")
- return shutil.which(cc) or cc
-
-def _module():
- from torch.utils.cpp_extension import load
-
- extra_cuda_cflags = ["-O3", "--expt-relaxed-constexpr"]
- host_cxx = _host_compiler()
- if host_cxx is not None:
- # Point both nvcc's host pass (-ccbin) and torch's C++ compile (CXX) at a
- # libtorch/nvcc-compatible compiler. Force (not setdefault): the system
- # default (CXX unset -> g++) can be a gcc too new for the torch headers.
- cxx_path = shutil.which(host_cxx) or host_cxx
- extra_cuda_cflags += ["-ccbin", cxx_path]
- os.environ["CXX"] = cxx_path
- os.environ["CC"] = _c_compiler_for(cxx_path)
-
- # gguf_kernel.cu carries its own PYBIND11_MODULE (appended at the end), so a
- # plain `load` of the single source compiles + binds the ggml_* ops.
- return load(
- name="freetoken_gguf_kernels",
- sources=[str(_CSRC / "gguf_kernel.cu")],
- extra_include_paths=[str(_CSRC)],
- extra_cuda_cflags=extra_cuda_cflags,
- verbose=True,
- )
-
-
-# ---- thin typed wrappers (signatures mirror sgl_kernel.quantization.gguf) ----
-
-
-def ggml_dequantize(
- weight: torch.Tensor, quant_type: int, m: int, n: int, dtype: torch.dtype | None = None
-) -> torch.Tensor:
- """Dequantize a packed GGUF weight ``[m, row_bytes]`` to a dense ``[m, n]`` tensor."""
- return _module().ggml_dequantize(weight, quant_type, m, n, dtype)
-
-
-def ggml_mul_mat_vec_a8(
- weight: torch.Tensor, x: torch.Tensor, quant_type: int, row: int
-) -> torch.Tensor:
- """MMVQ: small-batch GEMV with on-the-fly dequant. ``row`` = output features."""
- return _module().ggml_mul_mat_vec_a8(weight, x, quant_type, row)
-
-
-def ggml_mul_mat_a8(
- weight: torch.Tensor, x: torch.Tensor, quant_type: int, row: int
-) -> torch.Tensor:
- """MMQ: large-batch quantized matmul. ``row`` = output features."""
- return _module().ggml_mul_mat_a8(weight, x, quant_type, row)
-
-
-def ggml_moe_a8(
- x: torch.Tensor,
- weight: torch.Tensor,
- sorted_token_ids: torch.Tensor,
- expert_ids: torch.Tensor,
- num_tokens_post_padded: torch.Tensor,
- quant_type: int,
- row: int,
- top_k: int,
- tokens: int,
-) -> torch.Tensor:
- """MMQ grouped expert matmul over stacked experts ``weight[E, row, *]``."""
- return _module().ggml_moe_a8(
- x, weight, sorted_token_ids, expert_ids, num_tokens_post_padded,
- quant_type, row, top_k, tokens,
- )
-
-
-def ggml_moe_a8_vec(
- x: torch.Tensor,
- weight: torch.Tensor,
- topk_ids: torch.Tensor,
- top_k: int,
- quant_type: int,
- row: int,
- tokens: int,
-) -> torch.Tensor:
- """MMVQ grouped expert GEMV over stacked experts ``weight[E, row, *]``."""
- return _module().ggml_moe_a8_vec(x, weight, topk_ids, top_k, quant_type, row, tokens)
-
-
-def ggml_moe_get_block_size(quant_type: int) -> int:
- return _module().ggml_moe_get_block_size(quant_type)
-
-
-__all__ = [
- "ggml_dequantize",
- "ggml_mul_mat_vec_a8",
- "ggml_mul_mat_a8",
- "ggml_moe_a8",
- "ggml_moe_a8_vec",
- "ggml_moe_get_block_size",
-]
diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.py
index cd7a402a..e38a6cae 100644
--- a/python/freetoken/layers/activation.py
+++ b/python/freetoken/layers/activation.py
@@ -6,10 +6,17 @@ if TYPE_CHECKING:
import torch
-def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
+def _use_flashinfer(x: torch.Tensor, out: torch.Tensor | None) -> bool:
+ """flashinfer's *_and_mul kernels take fp16 / bf16 in and the same dtype out; the Triton kernels take any float dtypes."""
+ import torch
+
from freetoken.kernel.backend import is_flashinfer_installed
- if is_flashinfer_installed():
+ return x.dtype in (torch.float16, torch.bfloat16) and (out is None or out.dtype == x.dtype) and is_flashinfer_installed()
+
+
+def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
+ if _use_flashinfer(x, out):
from flashinfer import silu_and_mul
else:
from freetoken.kernel.triton.activation import silu_and_mul
@@ -18,9 +25,7 @@ def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
def gelu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
- from freetoken.kernel.backend import is_flashinfer_installed
-
- if is_flashinfer_installed():
+ if _use_flashinfer(x, out):
from flashinfer import gelu_and_mul
else:
from freetoken.kernel.triton.activation import gelu_and_mul
@@ -30,9 +35,7 @@ def gelu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
"""tanh-approximate GELU gate (`gelu_pytorch_tanh`) followed by elementwise mul."""
- from freetoken.kernel.backend import is_flashinfer_installed
-
- if is_flashinfer_installed():
+ if _use_flashinfer(x, out):
from flashinfer import gelu_tanh_and_mul
else:
from freetoken.kernel.triton.activation import gelu_tanh_and_mul
diff --git a/python/freetoken/layers/embedding.py b/python/freetoken/layers/embedding.py
index a2221654..f86670b1 100644
--- a/python/freetoken/layers/embedding.py
+++ b/python/freetoken/layers/embedding.py
@@ -3,7 +3,6 @@ from __future__ import annotations
from typing import Dict
import torch
-import torch.nn.functional as F
from freetoken.core import get_global_ctx
from freetoken.distributed import DistributedCommunicator, get_tp_info
from freetoken.utils import div_ceil, nvtx_annotate
@@ -13,11 +12,18 @@ from .quantization import LayerKind, QuantConfig, quant_method_for
class VocabParallelEmbedding(BaseOP):
+ """The vocab table, sharded over TP: its ``quant_method`` owns the table, looks rows up, and serves a tied LM head's product."""
+
+ quant_layer_kind = LayerKind.EMBEDDING
+
def __init__(
self,
num_embeddings: int,
embedding_dim: int,
embed_scale: float | None = None,
+ *,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
super().__init__()
tp_info = get_tp_info()
@@ -28,7 +34,12 @@ class VocabParallelEmbedding(BaseOP):
start_idx = self.num_embeddings_tp * tp_rank
finish_idx = min(start_idx + self.num_embeddings_tp, num_embeddings)
self.vocab_range = (start_idx, finish_idx - start_idx)
- self.weight = torch.empty(self.num_embeddings_tp, embedding_dim)
+ self.embedding_dim = embedding_dim
+ # the LM head subclass is a linear and builds its own method
+ self.quant_method = None
+ if self.quant_layer_kind is LayerKind.EMBEDDING:
+ self.quant_method = quant_method_for(quant_config, self, prefix)
+ self.quant_method.create_weights(self)
# Gemma scales embeddings by sqrt(hidden_size). The scale is materialized in
# the weight dtype (bf16) to match HF, which downcasts the scalar. The GPU
# scalar is built lazily (model __init__ runs on the meta device) and cached
@@ -39,14 +50,7 @@ class VocabParallelEmbedding(BaseOP):
@nvtx_annotate("Embedding")
def forward(self, x: torch.Tensor) -> torch.Tensor:
- from freetoken.kernel import indexing
-
- y = indexing(
- weights=self.weight,
- indices=x,
- vocab_range=self.vocab_range if self.tp_size > 1 else None,
- )
-
+ y = self.quant_method.embed(self, x)
if self.tp_size > 1:
y = self._comm.all_reduce(y)
if self._embed_scale is not None:
@@ -83,7 +87,6 @@ class ParallelLMHead(VocabParallelEmbedding):
self.in_features = embedding_dim
self.out_features = self.num_embeddings_tp
self.output_sizes = (self.num_embeddings_tp,)
- self.quant_method = None
if tied_embedding is None:
self.quant_method = quant_method_for(quant_config, self, prefix)
self.quant_method.create_weights(self)
@@ -132,7 +135,7 @@ class ParallelLMHead(VocabParallelEmbedding):
del indices
if self.tied_embedding is not None:
- logits = F.linear(x, self.tied_embedding.weight, self.bias)
+ logits = self.tied_embedding.quant_method.logits(self.tied_embedding, x, self.bias)
else:
logits = self.quant_method.apply(self, x)
if self.tp_size == 1:
diff --git a/python/freetoken/layers/gguf.py b/python/freetoken/layers/gguf.py
deleted file mode 100644
index ac49b1a5..00000000
--- a/python/freetoken/layers/gguf.py
+++ /dev/null
@@ -1,128 +0,0 @@
-"""Native-GGUF quantized layers: weights stay in their packed block layout and are
-dequantized *inside* the borrowed llama.cpp kernels (no bf16 copy ever materialized).
-
-Mirrors vLLM/sglang's ``GGUFLinearMethod`` / ``GGUFEmbeddingMethod`` dispatch, ported
-onto FreeToken's ``BaseOP``. FreeToken keeps fused projections (qkv, gate_up) as a
-single tensor: because Q4_0/K-quants pack each *output row* independently over the
-input dim, the loader can concatenate the per-shard packed rows along dim 0 (they
-share an input dim, hence the same ``row_bytes``), so a fused layer is still one
-``[out, row_bytes]`` qweight -- no per-shard padding bookkeeping needed.
-
-TP is assumed to be 1 (the gemma4 GGUF path restricts to TP=1, like the HF path).
-"""
-
-from __future__ import annotations
-
-import torch
-
-from freetoken.models.gguf.dequant import (
- BLOCK_SHAPE,
- GGML_BF16,
- GGML_F16,
- GGML_F32,
- GGML_NAME,
- GGML_Q4_0,
- GGML_Q6_K,
- GGML_Q8_0,
- row_bytes,
-)
-
-from .base import BaseOP
-
-# ggml type groups for kernel dispatch (subset we build kernels for).
-_UNQUANTIZED = {GGML_F32, GGML_F16, GGML_BF16}
-# standard + k-quants: both an MMVQ (small-batch GEMV) and MMQ (large-batch) kernel exist.
-_MMVQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K}
-_MMQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K}
-_DEQUANT = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K}
-
-# Below this token count, the MMVQ GEMV kernel wins (matches vLLM's heuristic).
-_MMVQ_SAFE = 6
-
-
-def fused_mul_mat_gguf(x: torch.Tensor, qweight: torch.Tensor, qweight_type: int) -> torch.Tensor:
- """y = x @ dequant(qweight).T, dispatched by batch size and quant type."""
- from freetoken.kernel.gguf import (
- ggml_dequantize,
- ggml_mul_mat_a8,
- ggml_mul_mat_vec_a8,
- )
-
- out_features = qweight.shape[0]
- if x.shape[0] == 0:
- return x.new_empty((0, out_features))
- if qweight_type in _UNQUANTIZED:
- return x @ qweight.T
- if x.shape[0] <= _MMVQ_SAFE and qweight_type in _MMVQ:
- return ggml_mul_mat_vec_a8(qweight, x, qweight_type, out_features)
- if qweight_type in _MMQ:
- return ggml_mul_mat_a8(qweight, x, qweight_type, out_features)
- if qweight_type in _DEQUANT:
- block, type_size = BLOCK_SHAPE[qweight_type]
- in_features = qweight.shape[1] // type_size * block
- weight = ggml_dequantize(qweight, qweight_type, out_features, in_features, x.dtype)
- return x @ weight.T
- raise NotImplementedError(f"unsupported GGUF type {GGML_NAME.get(qweight_type, qweight_type)}")
-
-
-class GGUFLinear(BaseOP):
- """Linear whose weight is a native GGUF block-quantized ``[out, row_bytes]`` tensor."""
-
- def __init__(
- self,
- in_features: int,
- out_features: int,
- quant_type: int,
- has_bias: bool = False,
- ):
- self.in_features = in_features
- self.out_features = out_features
- self._quant_type = quant_type
- self.qweight = torch.empty(out_features, row_bytes(in_features, quant_type), dtype=torch.uint8)
- self.bias = torch.empty(out_features) if has_bias else None
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- out = fused_mul_mat_gguf(x, self.qweight, self._quant_type)
- if self.bias is not None:
- out = out + self.bias
- return out
-
-
-class GGUFEmbedding(BaseOP):
- """Vocab embedding stored as a native GGUF block-quantized table.
-
- The full table is never dequantized: only the looked-up rows are gathered (in
- packed form) and dequantized per lookup, matching vLLM's ``_apply_gguf_embedding``.
- """
-
- def __init__(
- self,
- num_embeddings: int,
- embedding_dim: int,
- quant_type: int,
- embed_scale: float | None = None,
- ):
- self.num_embeddings = num_embeddings
- self.embedding_dim = embedding_dim
- self._quant_type = quant_type
- self.qweight = torch.empty(
- num_embeddings, row_bytes(embedding_dim, quant_type), dtype=torch.uint8
- )
- self._embed_scale = embed_scale
- self._embed_scale_t: torch.Tensor | None = None
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- from freetoken.kernel.gguf import ggml_dequantize
-
- flat = x.flatten()
- rows = self.qweight.index_select(0, flat) # [n, row_bytes] packed
- y = ggml_dequantize(rows, self._quant_type, flat.shape[0], self.embedding_dim, torch.bfloat16)
- y = y.view(*x.shape, self.embedding_dim)
- if self._embed_scale is not None:
- if self._embed_scale_t is None:
- self._embed_scale_t = torch.tensor(self._embed_scale, dtype=y.dtype, device=y.device)
- y = y * self._embed_scale_t
- return y
-
-
-__all__ = ["GGUFLinear", "GGUFEmbedding", "fused_mul_mat_gguf"]
diff --git a/python/freetoken/layers/linear.py b/python/freetoken/layers/linear.py
index d8a3e861..599ca049 100644
--- a/python/freetoken/layers/linear.py
+++ b/python/freetoken/layers/linear.py
@@ -63,6 +63,7 @@ class LinearReplicated(_LinearTPImpl):
output_size: int,
has_bias: bool,
*,
+ output_sizes: List[int] | None = None,
quant_config: QuantConfig | None = None,
prefix: str = "",
):
@@ -72,6 +73,7 @@ class LinearReplicated(_LinearTPImpl):
local_isize=input_size,
local_osize=output_size,
has_bias=has_bias,
+ output_sizes=output_sizes,
quant_config=quant_config,
prefix=prefix,
)
diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.py
index 79ad9e3a..598bd119 100644
--- a/python/freetoken/layers/moe.py
+++ b/python/freetoken/layers/moe.py
@@ -81,16 +81,12 @@ class MoELayer(BaseOP):
self.strategy = strategy
self.decode_target = decode_target
self.prefix = prefix
- # offload layers without a quant config stay on the format-tag banks (GGUF q4_0)
- self.quant_method = None
- if quant_config is not None or allocate_experts:
- self.quant_method = quant_method_for(quant_config, self, prefix)
- if allocate_experts:
- self.quant_method.create_weights(self)
+ self.quant_method = quant_method_for(quant_config, self, prefix)
+ if allocate_experts:
+ self.quant_method.create_weights(self)
def finalize(self) -> None:
- if self.quant_method is not None:
- self.quant_method.finalize(self)
+ self.quant_method.finalize(self)
def _maybe_all_reduce(self, hidden_states: torch.Tensor) -> torch.Tensor:
if self.tp_size > 1:
@@ -103,7 +99,6 @@ class MoELayer(BaseOP):
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
- assert self.quant_method is not None
return self.quant_method.apply(
hidden_states, topk_weights, topk_ids, self.quant_method.resident_view(self),
layer=self, is_prefill=get_global_ctx().batch.is_prefill,
@@ -191,7 +186,7 @@ class OffloadMoELayer(MoELayer):
@property
def skips_inactive_slots(self) -> bool:
- return self.quant_method is not None and self.quant_method.kernel.supports_inactive_slots
+ return self.quant_method.kernel.supports_inactive_slots
def forward(
self,
@@ -321,7 +316,7 @@ class OffloadMoELayer(MoELayer):
assert executor is not None, "CPU MoE executor was not initialized"
# the kernel reads every route's slot: give the inactive ones slot 0, which holds a
# complete payload or the bank's zero fill (never uninitialized storage), times weight 0
- # with several quants this holds only because the cache zeroes staged slot 0 rows (zero_inactive_route_slots)
+ # with several quants this holds only because the cache zeroes its staging area after prefill
clamp = not self.skips_inactive_slots
gpu_slots, gpu_w, cpu_ids = cache.ensure_experts_hybrid(self.layer_id, topk_ids, topk_weights, clamp=clamp)
pending = executor.decode_submit(self.layer_id, hidden_states, topk_weights, cpu_ids)
@@ -402,7 +397,6 @@ class OffloadMoELayer(MoELayer):
# ------------------------------------------------------------------
# Kernel dispatch: ``views`` are the bank tensors the movement step produced (in bank registration order) and ``topk_ids`` already index their rows.
- # GGUF q4_0 experts still dispatch on the cache's format tag until they get a method.
# ------------------------------------------------------------------
def _expert_gemm(
@@ -417,27 +411,15 @@ class OffloadMoELayer(MoELayer):
alphas: tuple[torch.Tensor, torch.Tensor] | None,
is_prefill: bool,
) -> torch.Tensor:
- if self.quant_method is not None:
- from freetoken.moe.legacy_format import canonical_role # legacy_format imports this package
+ from freetoken.moe.legacy_format import canonical_role # legacy_format imports this package
- view = ExpertView(
- {canonical_role(name): t for name, t in zip(cache.bank_schema, views)},
- slots=None if n is not None else topk_ids, n=n, alphas=alphas,
- )
- return self.quant_method.apply(
- hidden_states, topk_weights, topk_ids, view, layer=self, is_prefill=is_prefill
- )
- fmt = cache.quant_format
- if fmt == "q4_0":
- # Native GGUF Q4_0 experts: dequant-in-kernel grouped GEMV (MMVQ) over the
- # streamed packed banks; topk_ids already index the cache slots / layer.
- from freetoken.moe.fused_q4_0 import fused_experts_gguf_q4_0
-
- gate_up, down = views
- return fused_experts_gguf_q4_0(
- hidden_states, gate_up, down, topk_weights, topk_ids, self.activation
- )
- raise AssertionError(f"offload experts without a quant method only serve q4_0 banks, got {fmt!r}")
+ view = ExpertView(
+ {canonical_role(name): t for name, t in zip(cache.bank_schema, views)},
+ slots=None if n is not None else topk_ids, n=n, alphas=alphas,
+ )
+ return self.quant_method.apply(
+ hidden_states, topk_weights, topk_ids, view, layer=self, is_prefill=is_prefill
+ )
def iter_moe_layers(model) -> Iterator[MoELayer]:
diff --git a/python/freetoken/layers/quantization/__init__.py b/python/freetoken/layers/quantization/__init__.py
index 62019fc5..905b80ef 100644
--- a/python/freetoken/layers/quantization/__init__.py
+++ b/python/freetoken/layers/quantization/__init__.py
@@ -1,6 +1,7 @@
from .scheme import GroupShape, QuantKind, QuantScheme, WeightDesc
from .registry import LayerKind, dialects, method_class, methods_for, register_dialect, register_method
from .method import KernelSelectionError, QuantMethod, finalize_quant, select_kernel
+from .embedding import EmbeddingConfig, EmbeddingKernel, EmbeddingMethod, GGUFEmbeddingMethod, UnquantizedEmbeddingMethod
from .linear import (
Fp8BlockLinearMethod,
Fp8TensorLinearMethod,
@@ -28,6 +29,7 @@ from .names import NameMap
from .configs import (
CompressedTensorsConfig,
Fp8BlockConfig,
+ GGUFConfig,
ModelOptConfig,
Mxfp4Config,
NoQuantConfig,
@@ -42,11 +44,12 @@ __all__ = [
"QuantKind", "GroupShape", "QuantScheme", "WeightDesc",
"LayerKind", "dialects", "method_class", "methods_for", "register_dialect", "register_method",
"KernelSelectionError", "select_kernel", "QuantMethod",
+ "EmbeddingConfig", "EmbeddingKernel", "EmbeddingMethod", "UnquantizedEmbeddingMethod", "GGUFEmbeddingMethod",
"LinearConfig", "LinearKernel", "LinearMethod",
"UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod",
"BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod",
"UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod",
"QuantBackend", "set_quant_backend", "get_quant_backend", "NameMap",
- "QuantConfig", "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config",
+ "QuantConfig", "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "GGUFConfig", "Mxfp4Config",
"quant_method_for", "quantization_config_of", "set_quant_config", "get_quant_config", "finalize_quant",
]
diff --git a/python/freetoken/layers/quantization/configs/__init__.py b/python/freetoken/layers/quantization/configs/__init__.py
index 4bd7a6a9..358e6fec 100644
--- a/python/freetoken/layers/quantization/configs/__init__.py
+++ b/python/freetoken/layers/quantization/configs/__init__.py
@@ -1,11 +1,12 @@
"""One QuantConfig per checkpoint dialect; importing the package registers them all."""
from .base import NoQuantConfig, QuantConfig, quantization_config_of
-from . import compressed_tensors, fp8, modelopt, mxfp4
+from . import compressed_tensors, fp8, gguf, modelopt, mxfp4
from .compressed_tensors import CompressedTensorsConfig
from .fp8 import Fp8BlockConfig
from .modelopt import ModelOptConfig
from .mxfp4 import Mxfp4Config
+from .gguf import GGUFConfig
_NO_QUANT = NoQuantConfig()
@@ -33,6 +34,6 @@ def get_quant_config() -> QuantConfig | None:
__all__ = [
"QuantConfig", "quantization_config_of", "quant_method_for", "set_quant_config", "get_quant_config",
- "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config",
- "compressed_tensors", "fp8", "modelopt", "mxfp4",
+ "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config", "GGUFConfig",
+ "compressed_tensors", "fp8", "gguf", "modelopt", "mxfp4",
]
diff --git a/python/freetoken/layers/quantization/configs/base.py b/python/freetoken/layers/quantization/configs/base.py
index a6413610..78818e80 100644
--- a/python/freetoken/layers/quantization/configs/base.py
+++ b/python/freetoken/layers/quantization/configs/base.py
@@ -6,14 +6,15 @@ from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
+from ..embedding import EmbeddingConfig
from ..linear import LinearConfig
from ..moe import MoEConfig
from ..names import Matcher, NameMap, name_set
from ..quant_backend import get_quant_backend
-from ..registry import LayerKind, dialects, method_class, register_dialect
+from ..registry import LayerKind, dialects, method_class, methods_for, register_dialect
from ..scheme import QuantKind, QuantScheme
-LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig}
+LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig, LayerKind.EMBEDDING: EmbeddingConfig}
def cfg_get(cfg: Any, key: str, default: Any = None) -> Any:
@@ -88,6 +89,9 @@ class QuantConfig(ABC):
scheme = self.scheme_for(prefix)
layer_kind = layer.quant_layer_kind
kind = scheme.kind if scheme else QuantKind.NONE
+ if layer_kind is LayerKind.EMBEDDING and kind not in methods_for(layer_kind):
+ # a checkpoint that quantizes its linears stores the vocab table dense
+ scheme, kind = None, QuantKind.NONE
cls = method_class(kind, layer_kind)
cfg = self.layer_config(layer, layer_kind, scheme)
return cls(cfg, get_quant_backend().select(layer_kind, kind))
diff --git a/python/freetoken/layers/quantization/configs/gguf.py b/python/freetoken/layers/quantization/configs/gguf.py
new file mode 100644
index 00000000..bbc24f13
--- /dev/null
+++ b/python/freetoken/layers/quantization/configs/gguf.py
@@ -0,0 +1,82 @@
+"""GGUF: every tensor carries its own ggml type, read from the file's tensor table."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, ClassVar
+
+from ..ggml_types import is_quantized, type_name
+from ..names import NameMap
+from ..quant_backend import get_quant_backend
+from ..registry import LayerKind, method_class, register_dialect
+from ..scheme import QuantKind, QuantScheme, gguf_scheme
+from .base import QuantConfig, Stored
+
+
+@register_dialect
+class GGUFConfig(QuantConfig):
+ """``tensor_types`` is the file's name -> ggml type table; ``to_gguf(name, tensor_types)`` (the family's adapter) maps a checkpoint module name to the GGUF tensors that store it."""
+
+ dialect = "gguf"
+ STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {QuantKind.GGUF: {"weight": "weight"}}
+
+ def __init__(
+ self,
+ tensor_types: dict[str, int],
+ to_gguf: Callable[[str, dict[str, int]], tuple[str, ...]],
+ name_map: NameMap | None = None,
+ unquantized: tuple[str, ...] = (),
+ ):
+ super().__init__(name_map, unquantized)
+ self.tensor_types = tensor_types
+ self.to_gguf = to_gguf
+
+ @classmethod
+ def claims(cls, q: dict[str, Any]) -> bool:
+ return False
+
+ def _part_types(self, name: str) -> tuple[int, ...]:
+ types = []
+ for tensor in self.to_gguf(name, self.tensor_types):
+ key = f"{tensor}.weight" if f"{tensor}.weight" in self.tensor_types else tensor
+ if key not in self.tensor_types:
+ raise KeyError(f"GGUF has no tensor {key!r} for {name!r}")
+ types.append(self.tensor_types[key])
+ return tuple(types)
+
+ def scheme_for_name(self, name: str) -> QuantScheme | None:
+ types = self._part_types(name)
+ if not any(is_quantized(t) for t in types):
+ return None
+ if len(types) != 1:
+ raise NotImplementedError(f"{name} is stored as {len(types)} quantized GGUF tensors")
+ return gguf_scheme(types)
+
+ def scheme_for(self, prefix: str) -> QuantScheme | None:
+ if prefix in self._schemes:
+ return self._schemes[prefix]
+ names = self.name_map.to_checkpoint(prefix)
+ parts = [() if self.unquantized(n) else self._part_types(n) for n in names]
+ quantized = [any(is_quantized(t) for t in p) for p in parts]
+ if not any(quantized):
+ scheme = None
+ elif any(len(p) != 1 for p in parts):
+ found = {n: [type_name(t) for t in p] for n, p in zip(names, parts)}
+ raise NotImplementedError(f"fused module {prefix!r} needs one GGUF tensor per part, got {found}")
+ else:
+ scheme = gguf_scheme(p[0] for p in parts)
+ self._schemes[prefix] = scheme
+ return scheme
+
+ def get_quant_method(self, layer: Any, prefix: str):
+ if layer.quant_layer_kind is not LayerKind.MOE:
+ return super().get_quant_method(layer, prefix)
+ # every expert layer loads its GGUF banks, float experts included; the GGUF MoE kernel picks how to multiply them
+ parts = [self._part_types(n) for n in self.name_map.to_checkpoint(prefix)]
+ if any(len(p) != 1 for p in parts):
+ raise NotImplementedError(f"experts {prefix!r} are stored as {parts} GGUF tensors per projection")
+ scheme = gguf_scheme(p[0] for p in parts)
+ cfg = self.layer_config(layer, LayerKind.MOE, scheme)
+ return method_class(QuantKind.GGUF, LayerKind.MOE)(cfg, get_quant_backend().select(LayerKind.MOE, QuantKind.GGUF))
+
+
+__all__ = ["GGUFConfig"]
diff --git a/python/freetoken/layers/quantization/embedding/__init__.py b/python/freetoken/layers/quantization/embedding/__init__.py
new file mode 100644
index 00000000..ccdbece0
--- /dev/null
+++ b/python/freetoken/layers/quantization/embedding/__init__.py
@@ -0,0 +1,10 @@
+from .base import EmbeddingConfig, EmbeddingKernel, EmbeddingMethod
+from . import gguf, unquantized
+from .gguf import GGUFEmbeddingMethod
+from .unquantized import UnquantizedEmbeddingMethod
+
+__all__ = [
+ "EmbeddingConfig", "EmbeddingKernel", "EmbeddingMethod",
+ "UnquantizedEmbeddingMethod", "GGUFEmbeddingMethod",
+ "gguf", "unquantized",
+]
diff --git a/python/freetoken/layers/quantization/embedding/base.py b/python/freetoken/layers/quantization/embedding/base.py
new file mode 100644
index 00000000..caa873f4
--- /dev/null
+++ b/python/freetoken/layers/quantization/embedding/base.py
@@ -0,0 +1,64 @@
+"""The embedding side: per-layer config, the kernel ABC and the Method base. The method owns the vocab table: it looks rows up, and a tied LM head multiplies by the same table through it."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+import torch
+
+from ..method import QuantMethod
+from ..scheme import QuantScheme
+
+
+@dataclass(frozen=True)
+class EmbeddingConfig:
+ """The TP-local table shape, the dtype a lookup returns, and the table's scheme."""
+
+ num_embeddings: int
+ embedding_dim: int
+ tp_size: int
+ dtype: torch.dtype
+ scheme: QuantScheme | None = None
+
+ @classmethod
+ def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "EmbeddingConfig":
+ return cls(layer.num_embeddings_tp, layer.embedding_dim, layer.tp_size, torch.get_default_dtype(), scheme)
+
+
+class EmbeddingKernel(ABC):
+ """One backend for one table kind: the row lookup, and ``x @ table^T`` for a tied LM head."""
+
+ name: ClassVar[str]
+
+ def unusable_reason(self, cfg: EmbeddingConfig) -> str | None:
+ return None
+
+ def worth_it(self, cfg: EmbeddingConfig) -> bool:
+ return True
+
+ def finalize(self, layer: Any) -> None:
+ pass
+
+ @abstractmethod
+ def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor: ...
+
+ @abstractmethod
+ def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor: ...
+
+
+class EmbeddingMethod(QuantMethod):
+ """Declares the table for its kind; the lookup and the tied head's product go to the kernel."""
+
+ @abstractmethod
+ def create_weights(self, layer: Any) -> None: ...
+
+ def finalize(self, layer: Any) -> None:
+ self.kernel.finalize(layer)
+
+ def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:
+ return self.kernel.embed(layer, ids)
+
+ def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
+ return self.kernel.logits(layer, x, bias)
diff --git a/python/freetoken/layers/quantization/embedding/gguf.py b/python/freetoken/layers/quantization/embedding/gguf.py
new file mode 100644
index 00000000..f41b7b6a
--- /dev/null
+++ b/python/freetoken/layers/quantization/embedding/gguf.py
@@ -0,0 +1,45 @@
+"""GGUF tables: packed ggml rows, decoded per looked-up row; a tied head runs the GGUF linear kernel on the same rows."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+import torch
+
+from ..ggml_types import Segment, pad_rows, row_bytes
+from ..linear.gguf import GgmlLinearKernel
+from ..registry import LayerKind, register_method
+from ..scheme import QuantKind, gguf_types
+from .base import EmbeddingConfig, EmbeddingKernel, EmbeddingMethod
+
+
+class GgmlEmbeddingKernel(EmbeddingKernel):
+ name = "ggml"
+
+ def unusable_reason(self, cfg: EmbeddingConfig) -> str | None:
+ return "GGUF tables are not sharded over TP" if cfg.tp_size > 1 else None
+
+ def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel import ggml
+
+ cfg = layer.quant_method.cfg
+ (ggml_type,) = gguf_types(cfg.scheme)
+ rows = ggml.get_rows(layer.weight, ggml_type, cfg.embedding_dim, ids.reshape(-1).to(torch.int32), cfg.dtype)
+ return rows.view(*ids.shape, cfg.embedding_dim)
+
+ def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
+ cfg = layer.quant_method.cfg
+ (ggml_type,) = gguf_types(cfg.scheme)
+ table = SimpleNamespace(_ggml_segments=[Segment(ggml_type, 0, 1, 0, cfg.num_embeddings)], weight=layer.weight, bias=bias)
+ return GgmlLinearKernel().apply(table, x)
+
+
+@register_method(QuantKind.GGUF, LayerKind.EMBEDDING)
+class GGUFEmbeddingMethod(EmbeddingMethod):
+ candidates = (GgmlEmbeddingKernel,)
+
+ def create_weights(self, layer: Any) -> None:
+ (ggml_type,) = gguf_types(self.cfg.scheme)
+ h = self.cfg.embedding_dim
+ layer.weight = torch.empty(self.cfg.num_embeddings + pad_rows(h), row_bytes(ggml_type, h), dtype=torch.uint8)
diff --git a/python/freetoken/layers/quantization/embedding/unquantized.py b/python/freetoken/layers/quantization/embedding/unquantized.py
new file mode 100644
index 00000000..5b755baf
--- /dev/null
+++ b/python/freetoken/layers/quantization/embedding/unquantized.py
@@ -0,0 +1,33 @@
+"""Dense tables: the indexing kernel looks rows up; a tied head runs the unquantized linear kernel on the same table."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+import torch
+
+from ..linear.unquantized import TorchLinearKernel
+from ..registry import LayerKind, register_method
+from ..scheme import QuantKind
+from .base import EmbeddingKernel, EmbeddingMethod
+
+
+class IndexingEmbeddingKernel(EmbeddingKernel):
+ name = "indexing"
+
+ def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel import indexing
+
+ return indexing(weights=layer.weight, indices=ids, vocab_range=layer.vocab_range if layer.tp_size > 1 else None)
+
+ def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
+ return TorchLinearKernel().apply(SimpleNamespace(weight=layer.weight, bias=bias), x)
+
+
+@register_method(QuantKind.NONE, LayerKind.EMBEDDING)
+class UnquantizedEmbeddingMethod(EmbeddingMethod):
+ candidates = (IndexingEmbeddingKernel,)
+
+ def create_weights(self, layer: Any) -> None:
+ layer.weight = torch.empty(self.cfg.num_embeddings, self.cfg.embedding_dim)
diff --git a/python/freetoken/layers/quantization/ggml_types.py b/python/freetoken/layers/quantization/ggml_types.py
new file mode 100644
index 00000000..8b257cd8
--- /dev/null
+++ b/python/freetoken/layers/quantization/ggml_types.py
@@ -0,0 +1,88 @@
+"""ggml type facts shared by the GGUF dialect, its methods and the GGUF weight readers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+
+from freetoken.third_party import gguf
+
+# ggml reads up to this many values past the end of a packed matrix (llama.cpp MATRIX_ROW_PADDING)
+MATRIX_ROW_PADDING = 512
+
+UNQUANTIZED = frozenset(
+ int(t) for t in (gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16, gguf.GGMLQuantizationType.BF16)
+)
+
+
+_FLOAT_DTYPES = {
+ int(gguf.GGMLQuantizationType.F32): torch.float32,
+ int(gguf.GGMLQuantizationType.F16): torch.float16,
+ int(gguf.GGMLQuantizationType.BF16): torch.bfloat16,
+}
+
+
+def float_dtype(ggml_type: int) -> torch.dtype | None:
+ """The torch dtype of an unquantized ggml type, None for a quantized one."""
+ return _FLOAT_DTYPES.get(int(ggml_type))
+
+
+def type_name(ggml_type: int) -> str:
+ return gguf.GGMLQuantizationType(ggml_type).name
+
+
+def is_quantized(ggml_type: int) -> bool:
+ return int(ggml_type) not in UNQUANTIZED
+
+
+def row_bytes(ggml_type: int, k: int) -> int:
+ block, size = gguf.GGML_QUANT_SIZES[gguf.GGMLQuantizationType(ggml_type)]
+ if k % block:
+ raise ValueError(f"{type_name(ggml_type)} rows need a multiple of {block} values, got {k}")
+ return k // block * size
+
+
+def pad_rows(k: int) -> int:
+ """Zero rows to append after a packed [rows, K] matrix so ggml's tail reads stay inside it."""
+ return 0 if k % MATRIX_ROW_PADDING == 0 else 1
+
+
+def mmq_tail_bytes(ggml_type: int, k: int) -> int:
+ """Bytes past the last row of a packed [rows, K] matrix that MMQ reads and that must read as zeros."""
+ rem = k % MATRIX_ROW_PADDING
+ return 0 if rem == 0 or not is_quantized(ggml_type) else row_bytes(ggml_type, MATRIX_ROW_PADDING - rem)
+
+
+@dataclass(frozen=True)
+class Segment:
+ """A run of consecutive fused parts that share one ggml type: one packed matrix and one GEMM."""
+
+ ggml_type: int
+ first_part: int
+ num_parts: int
+ out_start: int
+ rows: int
+
+
+def segments(types: tuple[int, ...], output_sizes: tuple[int, ...]) -> list[Segment]:
+ if len(types) != len(output_sizes):
+ raise ValueError(f"{len(types)} ggml types for {len(output_sizes)} fused parts")
+ out: list[Segment] = []
+ start = 0
+ for i, (t, n) in enumerate(zip(types, output_sizes)):
+ if out and out[-1].ggml_type == t:
+ last = out[-1]
+ out[-1] = Segment(t, last.first_part, last.num_parts + 1, last.out_start, last.rows + n)
+ else:
+ out.append(Segment(int(t), i, 1, start, n))
+ start += n
+ return out
+
+
+def segment_param(index: int) -> str:
+ """State-dict leaf of segment ``index``: ``weight`` for the first, then ``weight_1``, ``weight_2``."""
+ return "weight" if index == 0 else f"weight_{index}"
+
+
+__all__ = ["MATRIX_ROW_PADDING", "Segment", "float_dtype", "is_quantized", "mmq_tail_bytes", "pad_rows", "row_bytes", "segment_param", "segments", "type_name"]
diff --git a/python/freetoken/layers/quantization/linear/__init__.py b/python/freetoken/layers/quantization/linear/__init__.py
index 8236abe0..781f2b45 100644
--- a/python/freetoken/layers/quantization/linear/__init__.py
+++ b/python/freetoken/layers/quantization/linear/__init__.py
@@ -1,13 +1,14 @@
from .base import LinearConfig, LinearKernel, LinearMethod
-from . import fp8_block, fp8_tensor, mxfp8, nvfp4, unquantized
+from . import fp8_block, fp8_tensor, gguf, mxfp8, nvfp4, unquantized
from .fp8_block import Fp8BlockLinearMethod
from .fp8_tensor import Fp8TensorLinearMethod
from .mxfp8 import Mxfp8LinearMethod
from .nvfp4 import Nvfp4LinearMethod
from .unquantized import UnquantizedLinearMethod
+from .gguf import GGUFLinearMethod
__all__ = [
"LinearConfig", "LinearKernel", "LinearMethod",
- "UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod",
- "fp8_block", "fp8_tensor", "mxfp8", "nvfp4", "unquantized",
+ "UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod", "GGUFLinearMethod",
+ "fp8_block", "fp8_tensor", "gguf", "mxfp8", "nvfp4", "unquantized",
]
diff --git a/python/freetoken/layers/quantization/linear/gguf.py b/python/freetoken/layers/quantization/linear/gguf.py
new file mode 100644
index 00000000..ac08dbe2
--- /dev/null
+++ b/python/freetoken/layers/quantization/linear/gguf.py
@@ -0,0 +1,56 @@
+"""GGUF Linear: packed ggml rows multiplied by llama.cpp's MMVQ / MMQ kernels; a fused module runs one GEMM per ggml type, float parts as plain bf16 rows."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+
+from ..ggml_types import is_quantized, pad_rows, row_bytes, segment_param, segments
+from ..registry import LayerKind, register_method
+from ..scheme import QuantKind, gguf_types
+from .base import LinearConfig, LinearKernel, LinearMethod
+
+
+
+class GgmlLinearKernel(LinearKernel):
+ name = "ggml"
+
+ def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ return None
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel import ggml
+
+ x2 = x.reshape(-1, x.shape[-1])
+ segs = layer._ggml_segments
+ m = x2.shape[0]
+ y = torch.empty(m, sum(s.rows for s in segs), dtype=x.dtype, device=x.device)
+ # every segment reads the same x: quantize it once for all the vector-kernel segments
+ xq = ggml.quantize_q8_1(x2) if sum(is_quantized(s.ggml_type) and ggml.uses_mmvq(s.ggml_type, m) for s in segs) > 1 else None
+ for i, s in enumerate(segs):
+ w = getattr(layer, segment_param(i))[: s.rows]
+ out = y[:, s.out_start : s.out_start + s.rows]
+ if is_quantized(s.ggml_type):
+ ggml.mul_mat(w, s.ggml_type, x2, out=out, xq=xq)
+ else:
+ torch.mm(x2, w.t(), out=out)
+ if layer.bias is not None:
+ y = y + layer.bias
+ return y.reshape(*x.shape[:-1], y.shape[-1])
+
+
+@register_method(QuantKind.GGUF, LayerKind.LINEAR)
+class GGUFLinearMethod(LinearMethod):
+ candidates = (GgmlLinearKernel,)
+
+ def create_weights(self, layer: Any) -> None:
+ g = self.cfg
+ segs = segments(gguf_types(g.scheme), g.output_sizes)
+ layer._ggml_segments = segs
+ k = g.in_features
+ for i, s in enumerate(segs):
+ if is_quantized(s.ggml_type):
+ setattr(layer, segment_param(i), torch.empty(s.rows + pad_rows(k), row_bytes(s.ggml_type, k), dtype=torch.uint8))
+ else:
+ setattr(layer, segment_param(i), torch.empty(s.rows, k))
diff --git a/python/freetoken/layers/quantization/moe/__init__.py b/python/freetoken/layers/quantization/moe/__init__.py
index a5f2dc66..1230f36f 100644
--- a/python/freetoken/layers/quantization/moe/__init__.py
+++ b/python/freetoken/layers/quantization/moe/__init__.py
@@ -1,13 +1,14 @@
from .base import BankSpec, ExpertView, MoEConfig, MoEKernel, MoEMethod
-from . import fp8_block, mxfp4, mxfp8, nvfp4, unquantized
+from . import fp8_block, gguf, mxfp4, mxfp8, nvfp4, unquantized
from .fp8_block import Fp8BlockMoEMethod
from .mxfp4 import Mxfp4MoEMethod
from .mxfp8 import Mxfp8MoEMethod
from .nvfp4 import Nvfp4MoEMethod
from .unquantized import UnquantizedMoEMethod
+from .gguf import GGUFMoEMethod
__all__ = [
"BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod",
- "UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod",
- "fp8_block", "mxfp4", "mxfp8", "nvfp4", "unquantized",
+ "UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod", "GGUFMoEMethod",
+ "fp8_block", "gguf", "mxfp4", "mxfp8", "nvfp4", "unquantized",
]
diff --git a/python/freetoken/layers/quantization/moe/gguf.py b/python/freetoken/layers/quantization/moe/gguf.py
new file mode 100644
index 00000000..78e051da
--- /dev/null
+++ b/python/freetoken/layers/quantization/moe/gguf.py
@@ -0,0 +1,118 @@
+"""GGUF experts: packed ggml rows in two banks (gate|up and down), multiplied by llama.cpp's mul_mat_id kernels; float experts run on the bf16 fused MoE."""
+
+from __future__ import annotations
+
+import math
+
+import torch
+
+from ..ggml_types import float_dtype, is_quantized, row_bytes, type_name
+from ..registry import LayerKind, register_method
+from ..scheme import QuantKind, gguf_types
+from .base import BankSpec, ExpertView, MoEConfig, MoEKernel, MoEMethod, gated_epilogue_reason, is_resident, limit_or_inf
+
+
+
+def fused_glu(layer) -> tuple[str, float] | None:
+ """The MMVQ epilogue (``ggml.mul_mat_id_glu``) that computes this layer's gated activation, or None to run it as a separate kernel."""
+ act, alpha, limit = layer.activation, float(layer.alpha), limit_or_inf(layer)
+ if act == "silu" and alpha == 1.0 and limit == math.inf:
+ return "swiglu", 0.0
+ if act == "gelu_tanh":
+ return "geglu", 0.0
+ if act == "swiglu_clamp" and alpha == 1.0 and limit != math.inf:
+ return "swiglu_clamp", limit
+ # ggml's fused swiglu_oai keeps its default alpha and limit
+ if act == "swigluoai" and math.isclose(alpha, 1.702, rel_tol=1e-6) and math.isclose(limit, 7.0, rel_tol=1e-6):
+ return "swiglu_oai", 0.0
+ return None
+
+
+def expert_types(cfg: MoEConfig) -> tuple[int, int]:
+ gate, up, down = gguf_types(cfg.scheme)
+ if gate != up:
+ raise NotImplementedError(f"GGUF experts with gate {type_name(gate)} and up {type_name(up)} need one gate_up bank type")
+ if is_quantized(gate) != is_quantized(down):
+ raise NotImplementedError(f"GGUF experts with gate_up {type_name(gate)} and down {type_name(down)} mix float and quantized rows")
+ return gate, down
+
+
+
+class GgmlMoEKernel(MoEKernel):
+ name = "ggml"
+ cpu_format = "gguf"
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ if not is_quantized(expert_types(cfg)[0]):
+ reason = gated_epilogue_reason(cfg)
+ if reason:
+ return f"float GGUF experts run on the bf16 fused MoE: {reason}"
+ return self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)
+
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
+ gate_up, down = expert_types(cfg)
+ i, h = cfg.local_intermediate, cfg.hidden
+ dtype = float_dtype(gate_up)
+ if dtype is not None:
+ return {"gate_up": BankSpec((2 * i, h), dtype), "down": BankSpec((h, i), dtype)}
+ return {
+ "gate_up": BankSpec((2 * i, row_bytes(gate_up, h)), torch.uint8),
+ "down": BankSpec((h, row_bytes(down, i)), torch.uint8),
+ }
+
+ def pack(self, pieces, cfg: MoEConfig, out):
+ i = cfg.local_intermediate
+ out["gate_up"][:, :i].copy_(pieces["gate"])
+ out["gate_up"][:, i:].copy_(pieces["up"])
+ out["down"].copy_(pieces["down"])
+ return {}
+
+ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
+ from freetoken.kernel import ggml
+ from freetoken.layers.activation import gated_act_and_mul
+
+ gate_up_type, down_type = expert_types(layer.quant_method.cfg)
+ if not is_quantized(gate_up_type):
+ return self._apply_float(layer, x, topk_weights, topk_ids, view, gate_up_type, is_prefill=is_prefill)
+ assert not layer.apply_router_weight_on_input, "GGUF experts apply the router weights on the output"
+ ids = topk_ids.to(torch.int32)
+ gate_up = view.tensors["gate_up"]
+ glu = fused_glu(layer)
+ small = x.shape[0] <= min(ggml.mmvq_max_tokens(gate_up_type), ggml.mmvq_max_tokens(down_type))
+ if glu is not None and small:
+ inter = ggml.mul_mat_id_glu(gate_up, gate_up_type, x, ids, *glu)
+ else:
+ h = ggml.mul_mat_id(gate_up, gate_up_type, x, ids)
+ t, k, two_i = h.shape
+ inter = torch.empty(t, k, two_i // 2, dtype=h.dtype, device=h.device)
+ gated_act_and_mul(layer.activation, h.view(-1, two_i), inter.view(-1, two_i // 2), alpha=float(layer.alpha), limit=limit_or_inf(layer))
+ del h # free the gate_up output before the down GEMM allocates its own
+ y = ggml.mul_mat_id(view.tensors["down"], down_type, inter, ids)
+ return ggml.moe_weighted_sum(y, topk_weights)
+
+ @staticmethod
+ def _apply_float(layer, x, topk_weights, topk_ids, view: ExpertView, ggml_type: int, *, is_prefill: bool):
+ """Float experts: their banks in their own dtype, through the bf16 fused MoE (the unquantized kernel's path)."""
+ from freetoken.moe.fused import fused_experts_decode_impl, fused_experts_impl
+
+ gate_up, down = view.tensors["gate_up"], view.tensors["down"]
+ if gate_up.dtype != x.dtype:
+ raise NotImplementedError(f"GGUF {type_name(ggml_type)} experts need {gate_up.dtype} activations, got {x.dtype}")
+ impl = fused_experts_impl if is_prefill or is_resident(layer) else fused_experts_decode_impl
+ return impl(x, gate_up, down, topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, float(layer.alpha), limit_or_inf(layer))
+
+
+@register_method(QuantKind.GGUF, LayerKind.MOE)
+class GGUFMoEMethod(MoEMethod):
+ candidates = (GgmlMoEKernel,)
+
+ @property
+ def cpu_format(self) -> str | None:
+ # the CPU executor reads Q4_0 experts in their GGUF rows and BF16 experts as its bf16 banks; other ggml types have no CPU kernel yet
+ gate_up, down = expert_types(self.cfg)
+ names = {type_name(gate_up), type_name(down)}
+ if names == {"Q4_0"}:
+ return "q4_0"
+ if names == {"BF16"}:
+ return "bf16"
+ return self.kernel.cpu_format
diff --git a/python/freetoken/layers/quantization/registry.py b/python/freetoken/layers/quantization/registry.py
index 2d13cf50..c561e55e 100644
--- a/python/freetoken/layers/quantization/registry.py
+++ b/python/freetoken/layers/quantization/registry.py
@@ -11,6 +11,7 @@ from .scheme import QuantKind
class LayerKind(Enum):
LINEAR = "linear"
MOE = "moe"
+ EMBEDDING = "embedding"
def __str__(self) -> str:
return self.value
diff --git a/python/freetoken/layers/quantization/scheme.py b/python/freetoken/layers/quantization/scheme.py
index 89f968af..ab43814e 100644
--- a/python/freetoken/layers/quantization/scheme.py
+++ b/python/freetoken/layers/quantization/scheme.py
@@ -20,6 +20,7 @@ class QuantKind(Enum):
MXFP8 = "mxfp8"
NVFP4 = "nvfp4"
MXFP4 = "mxfp4"
+ GGUF = "gguf"
def __str__(self) -> str:
return self.value
@@ -105,3 +106,13 @@ def nvfp4_scheme(*, input_scale: bool) -> QuantScheme:
def mxfp4_scheme() -> QuantScheme:
return QuantScheme(QuantKind.MXFP4, WeightDesc("e2m1", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale"})
+
+
+def gguf_scheme(types: Iterable[int]) -> QuantScheme:
+ """Packed ggml blocks, one ggml type per fused part (``types`` follows the parts' order)."""
+ return QuantScheme(QuantKind.GGUF, WeightDesc("ggml:" + ",".join(str(int(t)) for t in types), None, None), {"weight"})
+
+
+def gguf_types(scheme: QuantScheme) -> tuple[int, ...]:
+ assert scheme.kind is QuantKind.GGUF, scheme
+ return tuple(int(t) for t in scheme.weight.elem.removeprefix("ggml:").split(","))
diff --git a/python/freetoken/models/gemma4/__init__.py b/python/freetoken/models/gemma4/__init__.py
index 9f41f080..4fcf15a3 100644
--- a/python/freetoken/models/gemma4/__init__.py
+++ b/python/freetoken/models/gemma4/__init__.py
@@ -1,11 +1,6 @@
from .attention import Gemma4Attention
from .config import parse_config
-from .gguf import (
- dummy_q4_0_expert_sources,
- iter_gguf_weights,
- load_q4_0_expert_sources,
- parse_gguf_config,
-)
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import Gemma4ForCausalLM, Gemma4ForConditionalGeneration, Gemma4UnifiedForConditionalGeneration
from .moe import Gemma4Router
from .vision import Gemma4MultimodalEmbedder, Gemma4UnifiedVisionEmbedder, Gemma4VisionModel
@@ -32,6 +27,5 @@ __all__ = [
"iter_weights_parallel",
"parse_gguf_config",
"iter_gguf_weights",
- "load_q4_0_expert_sources",
- "dummy_q4_0_expert_sources",
+ "to_gguf",
]
diff --git a/python/freetoken/models/gemma4/gguf.py b/python/freetoken/models/gemma4/gguf.py
index 437822b5..95f89c3c 100644
--- a/python/freetoken/models/gemma4/gguf.py
+++ b/python/freetoken/models/gemma4/gguf.py
@@ -6,12 +6,13 @@ final-logit softcap), so this produces the *same* ``ModelConfig`` as
``gemma4.config.parse_config`` -- only the source is GGUF KV metadata instead of a
HF config object. transformers' own GGUF->config conversion is rejected by the
gemma4 strict dataclass (per-layer ``num_key_value_heads`` array), so we read the
-metadata directly. ``expert_quant`` is set to ``"q4_0"`` to route the routed experts
-through the native-Q4_0 offload-cache path.
+metadata directly. Every tensor keeps its packed ggml type (``QuantKind.GGUF``); the
+routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``).
"""
from __future__ import annotations
+import re
from typing import TYPE_CHECKING, Iterator
import torch
@@ -22,7 +23,8 @@ from freetoken.models.config import (
RotaryConfig,
SWAAttentionGroupConfig,
)
-from freetoken.models.gguf.dequant import GGML_Q4_0, GGML_Q6_K, dequantize, row_bytes
+from freetoken.models.gguf.reader import gguf_tensor_index
+from freetoken.models.gguf.tensors import float_tensor, fused, matrix
if TYPE_CHECKING:
from freetoken.models.gguf.config import GgufConfigShim
@@ -36,12 +38,9 @@ def _full_rotary_dim(shim: "GgufConfigShim", full_head_dim: int) -> int:
which drives the unrotated tail's frequencies to zero (ggml ops.cpp ``theta / ff``).
Count the untouched entries to get the rotated width back.
"""
- from freetoken.models.gguf.reader import iter_gguf_tensors
-
- for t in iter_gguf_tensors(shim.model_path):
- if t.name == "rope_freqs.weight": # F32, so the packed bytes ARE the values
- freqs = t.packed().reshape(-1).view(torch.float32)
- return int((freqs == 1.0).sum().item()) * 2
+ info = gguf_tensor_index(shim.model_path).get("rope_freqs.weight")
+ if info is not None:
+ return int((float_tensor(info) == 1.0).sum().item()) * 2
# A metadata-only GGUF (an FTW dir's source_metadata.gguf) has no tensor table. Every
# released gemma4 sets partial_rotary_factor 0.25 on the full layers, and the converter
# asserts rope_type == "proportional" (llama.cpp conversion/gemma.py).
@@ -112,8 +111,7 @@ def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:
model_type="gemma4",
architectures=list(shim.architectures),
moe_enabled=True,
- expert_quant="q4_0",
- moe_weight_format="q4_0",
+ expert_quant="gguf",
use_qk_norm=True,
attn_sm_scale=1.0,
final_logit_softcapping=float(g("final_logit_softcapping")),
@@ -144,8 +142,7 @@ def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:
# Weight loading: GGUF tensor names -> FreeToken gemma4 module params.
# --------------------------------------------------------------------------------------
-# Per-layer 1:1 norm/scale tensors (gguf suffix -> freetoken module-relative name). All
-# are tiny F32 tensors dequantized to bf16.
+# Per-layer 1:1 norm/scale tensors (gguf suffix -> freetoken module-relative name), all F32.
_LAYER_SCALAR_MAP = {
"attn_norm.weight": "input_layernorm.weight",
"attn_q_norm.weight": "self_attn.q_norm.weight",
@@ -161,14 +158,39 @@ _LAYER_SCALAR_MAP = {
"ffn_gate_inp.scale": "feed_forward.router.scale",
"ffn_down_exps.scale": "feed_forward.router.per_expert_scale",
}
-# Expert weight tensors (handled by the offload bank loader, not iter_gguf_weights).
-_EXPERT_SUFFIXES = ("ffn_gate_up_exps.weight", "ffn_down_exps.weight")
-
-def _to_bf16(t) -> torch.Tensor:
- """Dequantize a GgufTensor (F32/F16/Q*) to a dense bf16 tensor of its torch shape."""
- flat = dequantize(t.packed().reshape(-1), t.ggml_type, torch.bfloat16)
- return flat.reshape(t.shape)
+# checkpoint module leaf (gemma4 HF naming) -> GGUF tensor
+_LAYER_LEAVES = {
+ "self_attn.q_proj": "attn_q",
+ "self_attn.k_proj": "attn_k",
+ "self_attn.o_proj": "attn_output",
+ "mlp.gate_proj": "ffn_gate",
+ "mlp.up_proj": "ffn_up",
+ "mlp.down_proj": "ffn_down",
+ "experts.0.gate_proj": "ffn_gate_exps",
+ "experts.0.up_proj": "ffn_up_exps",
+ "experts.0.down_proj": "ffn_down_exps",
+}
+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name == "model.embed_tokens":
+ return ("token_embd",)
+ if name == "lm_head":
+ return ("output",) if "output.weight" in tensors else ()
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ blk, leaf = f"blk.{m.group(1)}", m.group(2)
+ if leaf == "self_attn.v_proj":
+ # the full-attention layers store no attn_v: k is reused as v
+ return (f"{blk}.attn_v",) if f"{blk}.attn_v.weight" in tensors else (f"{blk}.attn_k",)
+ if leaf in ("experts.0.gate_proj", "experts.0.up_proj") and f"{blk}.ffn_gate_up_exps.weight" in tensors:
+ return (f"{blk}.ffn_gate_up_exps",)
+ tensor = _LAYER_LEAVES.get(leaf)
+ return (f"{blk}.{tensor}",) if tensor is not None else ()
def _require_tp1(what: str) -> None:
@@ -191,284 +213,43 @@ def iter_gguf_weights(
include_moe_experts: bool,
include_non_moe: bool,
) -> Iterator[tuple[str, torch.Tensor]]:
- """Yield (param_name, tensor) for every non-expert gemma4 param.
-
- Quantized projections (attention q/k/v/o, shared-MLP gate/up/down) and the
- embedding stay in their native packed block layout and are yielded as ``.qweight``
- (uint8); norms / router / per-layer scalars dequantize to bf16. q/k/v and gate/up
- are fused by concatenating packed rows along the output dim (same input dim ->
- same ``row_bytes``). Routed experts are served from the offload cache, so they are
- skipped here (asserts the offload contract like the other MoE models).
- """
- from freetoken.models.gguf.reader import iter_gguf_tensors
+ """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""
+ if not include_non_moe:
+ return
from freetoken.utils import cached_load_hf_config
- assert not include_moe_experts, (
- "gemma4 GGUF stores experts as Q4_0 and only supports the offload backend; "
- "experts are loaded into the offload cache via load_q4_0_expert_sources()."
- )
- assert include_non_moe
_require_tp1("weight loading")
-
- # Full-attention layers ship no attn_v (k reused as v); SWA layers do. Knowing which
- # is which lets us emit the fused qkv as soon as its parts are present.
config = parse_gguf_config(cached_load_hf_config(model_path))
- k_eq_v_layers = {
- lid
- for lid in range(config.num_layers)
- if isinstance(config.attention_group_for_layer(lid), FullAttentionGroupConfig)
- and config.attention_group_for_layer(lid).k_eq_v
- }
-
- # Per-layer fusion buffers: layer -> {slot: packed[out, row_bytes]}.
- qkv_buf: dict[int, dict[str, torch.Tensor]] = {}
- gate_up_buf: dict[int, dict[str, torch.Tensor]] = {}
-
- def layer_of(name: str) -> int:
- return int(name.split(".")[1])
-
- for t in iter_gguf_tensors(model_path):
- name = t.name
- if name == "token_embd.weight":
- yield "model.embed_tokens.qweight", t.packed() # Q6_K packed table
- continue
- if name == "output_norm.weight":
- yield "model.norm.weight", _to_bf16(t)
- continue
- if name == "rope_freqs.weight":
- continue # rope frequencies recomputed in-engine
- if not name.startswith("blk."):
- continue
- if any(name.endswith(sfx) for sfx in _EXPERT_SUFFIXES):
- continue # routed experts -> offload banks
-
- layer = layer_of(name)
- suffix = name.split(".", 2)[2] # after "blk.N."
- base = f"model.layers.{layer}"
-
- if suffix in _LAYER_SCALAR_MAP:
- rel = _LAYER_SCALAR_MAP[suffix]
- tensor = _to_bf16(t)
- if rel.endswith("layer_scalar"):
- tensor = tensor.reshape(1)
- yield f"{base}.{rel}", tensor
- continue
-
- # Quantized projections: keep packed; fuse q/k/v and gate/up.
- if suffix == "attn_q.weight":
- qkv_buf.setdefault(layer, {})["q"] = t.packed()
- elif suffix == "attn_k.weight":
- qkv_buf.setdefault(layer, {})["k"] = t.packed()
- elif suffix == "attn_v.weight":
- qkv_buf.setdefault(layer, {})["v"] = t.packed()
- elif suffix == "ffn_gate.weight":
- gate_up_buf.setdefault(layer, {})["gate"] = t.packed()
- elif suffix == "ffn_up.weight":
- gate_up_buf.setdefault(layer, {})["up"] = t.packed()
- elif suffix == "attn_output.weight":
- yield f"{base}.self_attn.o_proj.qweight", t.packed()
- elif suffix == "ffn_down.weight":
- yield f"{base}.feed_forward.shared_mlp.down_proj.qweight", t.packed()
- else:
- raise ValueError(f"unmapped gemma4 GGUF tensor: {name}")
-
- # Emit fused qkv once all parts are present: k_eq_v full layers reuse k as v
- # (no attn_v tensor), SWA layers wait for their separate attn_v.
- slots = qkv_buf.get(layer)
- if slots is not None and "q" in slots and "k" in slots:
- if layer in k_eq_v_layers:
- v = slots["k"]
- elif "v" in slots:
- v = slots["v"]
- else:
- v = None
- if v is not None:
- yield f"{base}.self_attn.qkv_proj.qweight", torch.cat(
- [slots["q"], slots["k"], v], dim=0
- )
- del qkv_buf[layer]
- gu = gate_up_buf.get(layer)
- if gu is not None and "gate" in gu and "up" in gu:
- yield f"{base}.feed_forward.shared_mlp.gate_up_proj.qweight", torch.cat(
- [gu["gate"], gu["up"]], dim=0
- )
- del gate_up_buf[layer]
-
- assert not qkv_buf, f"incomplete qkv groups: {sorted(qkv_buf)}"
- assert not gate_up_buf, f"incomplete gate_up groups: {sorted(gate_up_buf)}"
-
-
-# --------------------------------------------------------------------------------------
-# Model layer swap: dense bf16 Linear/Embedding -> native GGUF-quant ops.
-# --------------------------------------------------------------------------------------
-
-
-def is_gguf_model(config: ModelConfig) -> bool:
- """True when the model was parsed from a GGUF checkpoint (native-quant path)."""
- return getattr(config, "moe_weight_format", None) == "q4_0"
-
-
-class GGUFTiedLMHead:
- """Tied LM head over a native Q6_K embedding table (logits via ggml matmul).
-
- Holds only a reference to the GGUF embedding (no params of its own -> empty
- state_dict), mirroring ``ParallelLMHead`` with ``tie_word_embeddings``. TP=1 only.
- """
-
- def __init__(self, embedding, quant_type: int):
- self._embedding = embedding
- self._quant_type = quant_type
-
- def state_dict(self, *, prefix: str = "", result=None):
- return result if result is not None else {}
-
- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False):
- state_dict.pop(f"{prefix}.weight", None)
- state_dict.pop(f"{prefix}.bias", None)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- from freetoken.core import get_global_ctx
- from freetoken.layers.gguf import fused_mul_mat_gguf
-
- batch = get_global_ctx().batch
- if batch.is_prefill:
- indices = batch.attn_metadata.get_last_indices(batch.size)
- x = x[indices].contiguous()
- return fused_mul_mat_gguf(x, self._embedding.qweight, self._quant_type)
-
-
-def convert_gemma4_to_gguf(model, config: ModelConfig) -> None:
- """In place: replace gemma4's dense projections + embedding with native GGUF ops.
-
- Quantized in the checkpoint -> swapped: attention qkv/o, shared-MLP gate_up/down
- (all Q4_0) and the token embedding (Q6_K, also the tied LM head). Left as dense
- bf16 (F32 in the GGUF): the router gate, all RMSNorms, the per-layer scalars, and
- the routed experts (served from the offload cache).
- """
- from freetoken.layers.gguf import GGUFEmbedding, GGUFLinear
-
- def swap_linear(owner, attr, quant_type=GGML_Q4_0):
- lin = getattr(owner, attr)
- out_features, in_features = lin.weight.shape
- setattr(
- owner,
- attr,
- GGUFLinear(in_features, out_features, quant_type, has_bias=lin.bias is not None),
- )
-
- inner = model.model
- embed = GGUFEmbedding(
- num_embeddings=config.vocab_size,
- embedding_dim=config.hidden_size,
- quant_type=GGML_Q6_K,
- embed_scale=config.embedding_scale,
- )
- inner.embed_tokens = embed
-
- for layer in inner.layers.op_list:
- swap_linear(layer.self_attn, "qkv_proj")
- swap_linear(layer.self_attn, "o_proj")
- swap_linear(layer.feed_forward.shared_mlp, "gate_up_proj")
- swap_linear(layer.feed_forward.shared_mlp, "down_proj")
-
- if config.tie_word_embeddings:
- model.lm_head = GGUFTiedLMHead(embed, GGML_Q6_K)
-
-
-# --------------------------------------------------------------------------------------
-# Routed-expert host banks (native Q4_0) for the offload cache.
-# --------------------------------------------------------------------------------------
-
-def _q4_0_expert_specs(config: ModelConfig) -> dict[str, tuple[tuple[int, ...], torch.dtype]]:
- E = config.num_experts
- H, I = config.hidden_size, config.moe_intermediate_size
- return {
- "gate_up": ((E, 2 * I, row_bytes(H, GGML_Q4_0)), torch.uint8),
- "down": ((E, H, row_bytes(I, GGML_Q4_0)), torch.uint8),
- }
-
-
-def load_q4_0_expert_sources(
- model_path: str, config: ModelConfig, *, layer_sink=None
-) -> dict[str, list[torch.Tensor]]:
- """Per-layer host banks of the routed experts' native Q4_0 block bytes.
-
- ``gate_up`` is one ``[E, 2I, H//32*18]`` tensor per layer and ``down`` one
- ``[E, H, I//32*18]`` per layer (independent :class:`HostBank` allocations) -- each
- expert's packed rows verbatim from the GGUF (no dequant), whole layers arriving in
- one shot (gate_up + down = 2 writes/layer), so the offload cache streams whole
- experts to the ggml MoE kernels.
-
- ``layer_sink=None`` (serving): pin each layer's two banks as they complete via an
- internally-owned :class:`PinPipeline` (or, on a CUDA-less host, allocate the mmap
- banks but never pin -- the CPU executor reads them pageable). ``layer_sink`` given
- (converter): the completion tracker fires into it instead -- nothing here is pinned,
- and the sink may release banks it has written out, so the returned tensors are only
- valid until then (the caller owns that tradeoff).
- """
- from freetoken.models.gguf.reader import iter_gguf_tensors
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks
-
- _require_tp1("expert banks")
- L, E = config.num_layers, config.num_experts
- H, I = config.hidden_size, config.moe_intermediate_size
- h_bytes, i_bytes = row_bytes(H, GGML_Q4_0), row_bytes(I, GGML_Q4_0)
- hb = alloc_layer_banks(_q4_0_expert_specs(config), L) # lazy anon mmaps (unpinned)
- banks = {name: [b.tensor for b in hb[name]] for name in hb}
- seen_gu, seen_dn = set(), set()
-
- def _load(sink) -> None:
- tracker = LayerCompletionTracker(2, hb, sink) if sink is not None else None # gate_up + down
- for t in iter_gguf_tensors(model_path):
- if not t.name.startswith("blk."):
- continue
- layer = int(t.name.split(".")[1])
- if t.name.endswith("ffn_gate_up_exps.weight"):
- banks["gate_up"][layer].copy_(t.packed().reshape(E, 2 * I, h_bytes))
- seen_gu.add(layer)
- elif t.name.endswith("ffn_down_exps.weight"):
- banks["down"][layer].copy_(t.packed().reshape(E, H, i_bytes))
- seen_dn.add(layer)
- else:
- continue
- if tracker is not None:
- tracker.note(layer)
-
- if layer_sink is not None:
- _load(layer_sink)
- elif torch.cuda.is_available():
- with PinPipeline() as pins:
- _load(pins)
- else:
- _load(None) # CUDA-less: mmap banks stay pageable, never pinned
-
- want = set(range(L))
- assert seen_gu == want and seen_dn == want, (
- f"missing Q4_0 expert layers: gate_up {sorted(want - seen_gu)}, "
- f"down {sorted(want - seen_dn)}"
- )
- return banks
-
-
-def dummy_q4_0_expert_sources(config: ModelConfig) -> dict[str, list[torch.Tensor]]:
- """Random Q4_0 expert banks shaped like ``load_q4_0_expert_sources`` output."""
- from freetoken.moe.host_banks import alloc_layer_banks, pin_banks
-
- L = config.num_layers
- hb = alloc_layer_banks(_q4_0_expert_specs(config), L)
- banks = {name: [b.tensor for b in hb[name]] for name in hb}
- for t in banks["gate_up"] + banks["down"]:
- t.random_(0, 256)
- if torch.cuda.is_available():
- pin_banks(hb) # match the other dummies: pin-after-fill (no-op mmap fill on CPU-only)
- return banks
+ index = gguf_tensor_index(model_path)
+ hidden = config.hidden_size
+
+ def t(name: str):
+ return index[name if name in index else f"{name}.weight"]
+
+ embed = t("token_embd")
+ yield from fused("model.embed_tokens", [embed], [matrix(embed)], hidden)
+ if "output.weight" in index:
+ out = t("output")
+ yield from fused("lm_head", [out], [matrix(out)], hidden)
+ yield "model.norm.weight", float_tensor(t("output_norm"))
+
+ for layer in range(config.num_layers):
+ blk, base = f"blk.{layer}", f"model.layers.{layer}"
+ for suffix, rel in _LAYER_SCALAR_MAP.items():
+ tensor = float_tensor(index[f"{blk}.{suffix}"])
+ yield f"{base}.{rel}", tensor.reshape(1) if rel.endswith("layer_scalar") else tensor
+ q, k = t(f"{blk}.attn_q"), t(f"{blk}.attn_k")
+ qkv = [q, k, index.get(f"{blk}.attn_v.weight", k)]
+ yield from fused(f"{base}.self_attn.qkv_proj", qkv, [matrix(p) for p in qkv], hidden)
+ o = t(f"{blk}.attn_output")
+ yield from fused(f"{base}.self_attn.o_proj", [o], [matrix(o)], int(o.ne[0]))
+ gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))
+ yield from fused(f"{base}.feed_forward.shared_mlp.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)
+ yield from fused(f"{base}.feed_forward.shared_mlp.down_proj", [down], [matrix(down)], int(down.ne[0]))
__all__ = [
"parse_gguf_config",
"iter_gguf_weights",
- "convert_gemma4_to_gguf",
- "is_gguf_model",
- "load_q4_0_expert_sources",
- "dummy_q4_0_expert_sources",
+ "to_gguf",
]
diff --git a/python/freetoken/models/gemma4/model.py b/python/freetoken/models/gemma4/model.py
index 5c91b905..1b9d0da7 100644
--- a/python/freetoken/models/gemma4/model.py
+++ b/python/freetoken/models/gemma4/model.py
@@ -61,6 +61,8 @@ class Gemma4Model(BaseOP):
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
embed_scale=config.embedding_scale,
+ quant_config=config.quant,
+ prefix=f"{prefix}.embed_tokens",
)
self.layers = OPList(
[
@@ -91,13 +93,6 @@ class Gemma4ForCausalLM(BaseLLMModel):
self._final_logit_softcapping = config.final_logit_softcapping
super().__init__()
- # GGUF checkpoints carry native block-quantized weights: swap the dense
- # projections + embedding for GGUF-quant ops (experts stay on the offload cache).
- from .gguf import convert_gemma4_to_gguf, is_gguf_model
-
- if is_gguf_model(config):
- convert_gemma4_to_gguf(self, config)
-
def forward(self) -> torch.Tensor:
output = self.model.forward(get_global_ctx().batch.input_ids)
logits = self.lm_head.forward(output)
diff --git a/python/freetoken/models/gemma4/weight.py b/python/freetoken/models/gemma4/weight.py
index ba990c00..08293c7e 100644
--- a/python/freetoken/models/gemma4/weight.py
+++ b/python/freetoken/models/gemma4/weight.py
@@ -196,7 +196,7 @@ def iter_weights(
continue
# Per-expert NVFP4 tensors go to the offload cache (expert pieces),
- # not this dense pass; fused bf16/q4_0 experts lack ".experts.<int>." so are unaffected.
+ # not this dense pass; fused bf16 experts lack ".experts.<int>." so are unaffected.
if _NVFP4_EXPERT_RE.search(raw_name):
continue
diff --git a/python/freetoken/models/gguf/__init__.py b/python/freetoken/models/gguf/__init__.py
index 75e40e4f..20a97b46 100644
--- a/python/freetoken/models/gguf/__init__.py
+++ b/python/freetoken/models/gguf/__init__.py
@@ -1,26 +1,19 @@
-from .dequant import GGML_NAME, dequantize
from .reader import (
FTW_METADATA_GGUF,
- GgufTensor,
gguf_architecture,
gguf_config_source,
gguf_tensor_names,
is_gguf_path,
- iter_gguf_tensors,
load_gguf_metadata,
write_metadata_gguf,
)
__all__ = [
- "GGML_NAME",
- "dequantize",
"FTW_METADATA_GGUF",
- "GgufTensor",
"gguf_architecture",
"gguf_config_source",
"gguf_tensor_names",
"is_gguf_path",
- "iter_gguf_tensors",
"load_gguf_metadata",
"write_metadata_gguf",
]
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index 63b1a18b..4eb2a999 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -18,6 +18,7 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names
# reuses the model classes but a GGUF parse_config / iter_weights).
GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
"gemma4": "Gemma4GGUFForCausalLM",
+ "qwen4exp": "Qwen4ExpGGUFForCausalLM",
}
diff --git a/python/freetoken/models/gguf/dequant.py b/python/freetoken/models/gguf/dequant.py
deleted file mode 100644
index 77c3ea01..00000000
--- a/python/freetoken/models/gguf/dequant.py
+++ /dev/null
@@ -1,153 +0,0 @@
-"""GGML block-quant dequantization in pure torch (the formats this repo's GGUF
-checkpoints use: Q4_0, Q6_K, plus trivial F32/F16/BF16).
-
-This is the *reference / CPU* path, NOT the engine's hot path: GGUF weights stay
-packed and are dequantized inside the borrowed ggml CUDA kernels (see
-``freetoken.kernel.gguf``). These routines are used only to (a) materialize the few
-dense F32/F16 tensors at load (norms, scales, router) via :func:`dequantize`, and
-(b) cross-check the CUDA kernels in tests. The ``BLOCK_SHAPE`` table and
-:func:`row_bytes` are the type metadata the packed (kernel) path also relies on.
-
-Each ``dequant_*`` takes the raw little-endian bytes as a ``uint8`` tensor whose
-final axis spans whole blocks, and returns the values in *storage order* (ggml's
-fastest axis first); the caller reshapes to the torch shape (``dims[::-1]``). The
-math mirrors ``ggml-quants.c``.
-"""
-
-from __future__ import annotations
-
-import torch
-
-# ggml_type enum values (subset present in these checkpoints).
-GGML_F32 = 0
-GGML_F16 = 1
-GGML_Q4_0 = 2
-GGML_Q8_0 = 8
-GGML_Q6_K = 14
-GGML_BF16 = 30
-
-# (block numel, bytes per block) per ggml type.
-BLOCK_SHAPE: dict[int, tuple[int, int]] = {
- GGML_F32: (1, 4),
- GGML_F16: (1, 2),
- GGML_BF16: (1, 2),
- GGML_Q4_0: (32, 18),
- GGML_Q8_0: (32, 34),
- GGML_Q6_K: (256, 210),
-}
-
-GGML_NAME = {
- GGML_F32: "F32",
- GGML_F16: "F16",
- GGML_BF16: "BF16",
- GGML_Q4_0: "Q4_0",
- GGML_Q8_0: "Q8_0",
- GGML_Q6_K: "Q6_K",
-}
-
-
-def row_bytes(numel: int, ggml_type: int) -> int:
- """Packed byte length of one row of ``numel`` elements in ``ggml_type`` blocks.
-
- Single source of truth for the ``numel // block * type_size`` math shared by the
- packed-weight ops (``GGUFLinear``/``GGUFEmbedding``) and the expert bank loaders.
- """
- block, type_size = BLOCK_SHAPE[ggml_type]
- assert numel % block == 0, (
- f"{numel} not a multiple of block {block} for {GGML_NAME.get(ggml_type, ggml_type)}"
- )
- return numel // block * type_size
-
-
-def _f16_scales(raw: torch.Tensor, lo: int, hi: int) -> torch.Tensor:
- """Reinterpret bytes ``[lo:hi]`` (2 per block) of each block row as fp16 -> fp32 [N,1]."""
- return raw[:, lo:hi].contiguous().view(torch.float16).to(torch.float32)
-
-
-def dequant_q4_0(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor:
- """Q4_0: per 32-elem block = fp16 scale ``d`` + 16 packed nibbles; ``w = d*(q-8)``.
-
- Byte ``j`` of the 16 holds element ``j`` in its low nibble and ``j+16`` in its high
- nibble, so storage order within the block is ``[lo0..lo15, hi0..hi15]``.
- """
- raw = raw.reshape(-1, 18)
- d = _f16_scales(raw, 0, 2) # [N,1]
- qs = raw[:, 2:18] # [N,16] uint8
- lo = (qs & 0x0F).to(torch.float32)
- hi = (qs >> 4).to(torch.float32)
- q = torch.cat([lo, hi], dim=1) # [N,32]
- return ((q - 8.0) * d).reshape(-1).to(out_dtype)
-
-
-def dequant_q6_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor:
- """Q6_K: 256-elem super-block = 128B low nibbles + 64B high 2-bits + 16 int8
- sub-scales + fp16 ``d``. Direct vectorization of ggml's two-half loop."""
- raw = raw.reshape(-1, 210)
- n = raw.shape[0]
- ql = raw[:, 0:128] # [n,128]
- qh = raw[:, 128:192] # [n,64]
- sc = raw[:, 192:208].view(torch.int8).to(torch.float32) # [n,16]
- d = _f16_scales(raw, 208, 210) # [n,1]
-
- y = torch.empty((n, 256), dtype=torch.float32, device=raw.device)
- # l in 0..15 -> is=0; l in 16..31 -> is=1 (per ggml: is = l/16).
- is_idx = (torch.arange(32, device=raw.device) // 16) # [32] in {0,1}
- for h in range(2): # two 128-elem halves of the super-block
- qlh = ql[:, h * 64:(h + 1) * 64] # [n,64]
- qhh = qh[:, h * 32:(h + 1) * 32] # [n,32]
- sch = sc[:, h * 8:(h + 1) * 8] # [n,8]
- a = qlh[:, 0:32].to(torch.int32) # ql[l]
- b = qlh[:, 32:64].to(torch.int32) # ql[l+32]
- hb = qhh.to(torch.int32) # qh[l]
- q1 = ((a & 0x0F) | (((hb >> 0) & 3) << 4)) - 32
- q2 = ((b & 0x0F) | (((hb >> 2) & 3) << 4)) - 32
- q3 = ((a >> 4) | (((hb >> 4) & 3) << 4)) - 32
- q4 = ((b >> 4) | (((hb >> 6) & 3) << 4)) - 32
- s1 = sch.index_select(1, is_idx + 0).to(torch.float32)
- s2 = sch.index_select(1, is_idx + 2).to(torch.float32)
- s3 = sch.index_select(1, is_idx + 4).to(torch.float32)
- s4 = sch.index_select(1, is_idx + 6).to(torch.float32)
- base = h * 128
- y[:, base + 0:base + 32] = d * s1 * q1.to(torch.float32)
- y[:, base + 32:base + 64] = d * s2 * q2.to(torch.float32)
- y[:, base + 64:base + 96] = d * s3 * q3.to(torch.float32)
- y[:, base + 96:base + 128] = d * s4 * q4.to(torch.float32)
- return y.reshape(-1).to(out_dtype)
-
-
-_DEQUANT = {
- GGML_Q4_0: dequant_q4_0,
- GGML_Q6_K: dequant_q6_k,
-}
-
-
-def dequantize(raw: torch.Tensor, ggml_type: int, out_dtype: torch.dtype) -> torch.Tensor:
- """Dequantize ``raw`` (uint8) of any supported ggml type to flat ``out_dtype``."""
- if ggml_type == GGML_F32:
- return raw.view(torch.float32).to(out_dtype)
- if ggml_type == GGML_F16:
- return raw.view(torch.float16).to(out_dtype)
- if ggml_type == GGML_BF16:
- return raw.view(torch.bfloat16).to(out_dtype)
- fn = _DEQUANT.get(ggml_type)
- if fn is None:
- raise NotImplementedError(
- f"dequant for ggml type {GGML_NAME.get(ggml_type, ggml_type)} not implemented"
- )
- return fn(raw, out_dtype)
-
-
-__all__ = [
- "GGML_F32",
- "GGML_F16",
- "GGML_BF16",
- "GGML_Q4_0",
- "GGML_Q8_0",
- "GGML_Q6_K",
- "GGML_NAME",
- "BLOCK_SHAPE",
- "row_bytes",
- "dequant_q4_0",
- "dequant_q6_k",
- "dequantize",
-]
diff --git a/python/freetoken/models/gguf/experts.py b/python/freetoken/models/gguf/experts.py
new file mode 100644
index 00000000..0da56ceb
--- /dev/null
+++ b/python/freetoken/models/gguf/experts.py
@@ -0,0 +1,104 @@
+"""Routed-expert banks straight from a GGUF's ``blk.N.ffn_{gate,up,down}_exps`` tensors, in their packed ggml rows: pinned host banks for the offload cache, or device banks for resident experts.
+
+Each layer keeps its own ggml types, so each layer gets its own bank shapes: ``gate_up`` [E, 2I, row_bytes] (expert e's gate rows, then its up rows) and ``down`` [E, H, row_bytes]; float experts keep their dtype ([E, 2I, H] and [E, H, I]).
+A GGUF that stores ``ffn_gate_up_exps`` (gemma4) already holds the gate_up rows in that order."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+
+import torch
+
+from freetoken.layers.quantization import QuantKind
+from freetoken.layers.quantization.ggml_types import float_dtype, mmq_tail_bytes, pad_rows, type_name
+from freetoken.utils import init_logger
+
+from .reader import GgufTensorInfo, gguf_tensor_index
+
+logger = init_logger(__name__)
+
+
+def _expert_rows(info: GgufTensorInfo) -> torch.Tensor:
+ """[E, rows, row_bytes] packed ggml rows, or [E, rows, K] for a float tensor."""
+ k, rows, experts = info.ne
+ raw = torch.from_numpy(info.data()).view(experts, rows, info.nbytes // (experts * rows))
+ dtype = float_dtype(info.ggml_type)
+ return raw if dtype is None else raw.view(dtype)
+
+
+def _drop_page_cache(info: GgufTensorInfo) -> None:
+ try:
+ fd = os.open(info.path, os.O_RDONLY)
+ try:
+ os.posix_fadvise(fd, info.offset, info.nbytes, os.POSIX_FADV_DONTNEED)
+ finally:
+ os.close(fd)
+ except OSError:
+ pass
+
+
+def _device_bank(shape: tuple[int, ...], k: int, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
+ """An [E, rows, row] device bank; a zero row follows the last one when ggml's MMQ tiles read past it."""
+ experts, rows, row = shape
+ flat = torch.empty(experts * rows + pad_rows(k), row, dtype=dtype, device=device)
+ flat[experts * rows :].zero_()
+ return flat[: experts * rows].view(shape)
+
+
+def load_gguf_expert_banks(model_path: str, num_layers: int, *, first_layer: int = 0, dummy: bool = False, device: torch.device | None = None):
+ """``ExpertBanks`` with one bank per (layer, role): pinned host banks for the offload cache, filled from the file and pinned while later layers load, or resident banks on ``device``."""
+ from freetoken.moe.expert_banks import ExpertBanks
+ from freetoken.moe.host_banks import HostBank, PinPipeline
+
+ index = gguf_tensor_index(model_path)
+ sources: dict[str, list[torch.Tensor]] = {"gate_up": [], "down": []}
+ tails: dict[str, list[int]] = {"gate_up": [], "down": []}
+ tags = []
+ with PinPipeline() if device is None else contextlib.nullcontext() as pins:
+ for layer in range(num_layers):
+ blk = f"blk.{first_layer + layer}"
+ down = index[f"{blk}.ffn_down_exps.weight"]
+ if f"{blk}.ffn_gate_up_exps.weight" in index:
+ gate = index[f"{blk}.ffn_gate_up_exps.weight"]
+ parts = [gate]
+ else:
+ gate, up = index[f"{blk}.ffn_gate_exps.weight"], index[f"{blk}.ffn_up_exps.weight"]
+ if gate.ggml_type != up.ggml_type:
+ raise NotImplementedError(f"{blk}: gate and up experts have different ggml types")
+ parts = [gate, up]
+ tags.append(("ggml", type_name(gate.ggml_type), type_name(down.ggml_type)))
+ tails["gate_up"].append(mmq_tail_bytes(gate.ggml_type, gate.ne[0]))
+ tails["down"].append(mmq_tail_bytes(down.ggml_type, down.ne[0]))
+ rows = [_expert_rows(info) for info in parts]
+ d = _expert_rows(down)
+ gate_up_shape = (rows[0].shape[0], sum(r.shape[1] for r in rows), rows[0].shape[2])
+ if device is None:
+ host = {"gate_up": HostBank(gate_up_shape, rows[0].dtype), "down": HostBank(tuple(d.shape), d.dtype)}
+ banks = {role: bank.tensor for role, bank in host.items()}
+ else:
+ banks = {
+ "gate_up": _device_bank(gate_up_shape, gate.ne[0], rows[0].dtype, device),
+ "down": _device_bank(tuple(d.shape), down.ne[0], d.dtype, device),
+ }
+ if dummy:
+ banks["gate_up"].zero_()
+ banks["down"].zero_()
+ else:
+ start = 0
+ for r in rows:
+ banks["gate_up"][:, start : start + r.shape[1]].copy_(r)
+ start += r.shape[1]
+ banks["down"].copy_(d)
+ for info in (*parts, down):
+ _drop_page_cache(info)
+ if device is None:
+ pins(layer, host)
+ for role, bank in banks.items():
+ sources[role].append(bank)
+ total = sum(t.numel() * t.element_size() for ts in sources.values() for t in ts)
+ logger.info_rank0(f"GGUF expert banks: {num_layers} layers, {total / 2**30:.1f} GiB {'on ' + str(device) if device is not None else 'in host RAM'}")
+ return ExpertBanks("gguf", sources, kind=QuantKind.GGUF, kernel="ggml", layout=None, layer_tags=tags, row_tail_bytes=tails)
+
+
+__all__ = ["load_gguf_expert_banks"]
diff --git a/python/freetoken/models/gguf/reader.py b/python/freetoken/models/gguf/reader.py
index b950d929..2715f129 100644
--- a/python/freetoken/models/gguf/reader.py
+++ b/python/freetoken/models/gguf/reader.py
@@ -2,22 +2,20 @@
Thin layer over ``gguf.GGUFReader`` (gguf-py). Metadata is read into a plain dict
keyed by the GGUF field name (``general.architecture``, ``gemma4.block_count`` ...);
-tensors are exposed as ``GgufTensor`` records carrying the *torch* shape (ggml dims
-reversed), the ggml quant type, and a zero-copy ``uint8`` view of the packed block
-bytes laid out as ``[rows, row_bytes]`` (rows = product of all but the fastest ggml
-dim; row_bytes spans whole quant blocks of the fastest dim).
+tensors are indexed as ``GgufTensorInfo`` records (ggml dims, ggml type, and where the
+packed bytes sit in the memory-mapped file).
"""
from __future__ import annotations
import functools
import os
+import re
import struct
from dataclasses import dataclass
-from typing import Any, Iterator
+from typing import Any
import numpy as np
-import torch
def is_gguf_path(model_path: str) -> bool:
@@ -65,7 +63,7 @@ def write_metadata_gguf(source_gguf: str, dest_path: str) -> None:
Validates by re-parsing: the copy must list zero tensors and expose the identical KV
key set (the KV *bytes* are copied verbatim, so identical keys imply identical values).
"""
- import gguf
+ from freetoken.third_party import gguf
reader = gguf.GGUFReader(source_gguf)
assert reader.tensors, f"{source_gguf}: no tensors to bound the KV section"
@@ -98,20 +96,6 @@ def write_metadata_gguf(source_gguf: str, dest_path: str) -> None:
)
-@dataclass(frozen=True)
-class GgufTensor:
- name: str
- shape: tuple[int, ...] # torch order (ggml dims reversed)
- ggml_type: int
- rows: int # product of shape[:-1] over the *ggml* layout = blocks-major rows
- row_bytes: int # packed bytes per row (whole quant blocks of the fastest dim)
- _raw: np.ndarray # uint8 view, shape [rows, row_bytes]
-
- def packed(self) -> torch.Tensor:
- """Zero-copy ``[rows, row_bytes]`` uint8 tensor of the native block bytes."""
- return torch.from_numpy(self._raw)
-
-
def _field_value(reader, name: str) -> Any:
field = reader.fields.get(name)
if field is None:
@@ -121,7 +105,7 @@ def _field_value(reader, name: str) -> Any:
@functools.cache
def _reader(model_path: str):
- import gguf
+ from freetoken.third_party import gguf
return gguf.GGUFReader(model_path)
@@ -133,6 +117,55 @@ def load_gguf_metadata(model_path: str) -> dict[str, Any]:
return {name: field.contents() for name, field in reader.fields.items()}
+_SPLIT_RE = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$")
+
+
+def gguf_split_paths(model_path: str) -> tuple[str, ...]:
+ """Every file of a split GGUF (``name-00001-of-00002.gguf``), in order; a single file is its own list."""
+ count = int(_field_value(_reader(model_path), "split.count") or 1)
+ if count == 1:
+ return (model_path,)
+ match = _SPLIT_RE.search(model_path)
+ if match is None:
+ raise ValueError(f"{model_path} is one of {count} GGUF splits but is not named like name-00001-of-{count:05d}.gguf")
+ stem = model_path[: match.start()]
+ paths = tuple(f"{stem}-{i:05d}-of-{count:05d}.gguf" for i in range(1, count + 1))
+ missing = [p for p in paths if not os.path.isfile(p)]
+ if missing:
+ raise FileNotFoundError(f"GGUF split files missing: {missing}")
+ return paths
+
+
+@dataclass(frozen=True)
+class GgufTensorInfo:
+ """Where one tensor's packed bytes sit: ``ne`` is in ggml order (fastest dim first)."""
+
+ name: str
+ path: str
+ offset: int
+ nbytes: int
+ ggml_type: int
+ ne: tuple[int, ...]
+
+ def data(self) -> np.ndarray:
+ """Zero-copy uint8 view of the bytes (memory-mapped)."""
+ reader = _reader(self.path)
+ return reader.data[self.offset : self.offset + self.nbytes]
+
+
+def gguf_tensor_index(model_path: str) -> dict[str, GgufTensorInfo]:
+ """Every tensor of every split, by name."""
+ index: dict[str, GgufTensorInfo] = {}
+ for path in gguf_split_paths(model_path):
+ for t in _reader(path).tensors:
+ index[t.name] = GgufTensorInfo(
+ t.name, path, int(t.data_offset), int(t.n_bytes), int(t.tensor_type), tuple(int(x) for x in t.shape)
+ )
+ return index
+
+
def gguf_architecture(model_path: str) -> str:
arch = _field_value(_reader(model_path), "general.architecture")
if arch is None:
@@ -140,39 +173,8 @@ def gguf_architecture(model_path: str) -> str:
return str(arch)
-def iter_gguf_tensors(model_path: str) -> Iterator[GgufTensor]:
- """Yield every tensor with its torch shape, ggml type, and packed block bytes."""
- import gguf
-
- reader = _reader(model_path)
- for t in reader.tensors:
- ne = [int(s) for s in t.shape] # ggml order, fastest dim first
- torch_shape = tuple(reversed(ne))
- block, type_size = gguf.GGML_QUANT_SIZES[t.tensor_type]
- n_fast = ne[0]
- if n_fast % block != 0:
- raise ValueError(
- f"{t.name}: fastest dim {n_fast} not a multiple of block {block} "
- f"for {t.tensor_type.name}"
- )
- row_bytes = n_fast // block * type_size
- rows = int(np.prod(ne[1:])) if len(ne) > 1 else 1
- # gguf-py returns quantized tensors as raw uint8 but F32/F16 as typed arrays;
- # normalize everything to a flat byte view before shaping into [rows, row_bytes].
- flat = np.ascontiguousarray(t.data).reshape(-1).view(np.uint8)
- raw = flat.reshape(rows, row_bytes)
- yield GgufTensor(
- name=t.name,
- shape=torch_shape,
- ggml_type=int(t.tensor_type),
- rows=rows,
- row_bytes=row_bytes,
- _raw=raw,
- )
-
-
def gguf_tensor_names(model_path: str) -> set[str]:
- return {t.name for t in _reader(model_path).tensors}
+ return {t.name for path in gguf_split_paths(model_path) for t in _reader(path).tensors}
__all__ = [
@@ -181,9 +183,10 @@ __all__ = [
"OUTPUT_WEIGHT_PRESENT_KV",
"gguf_config_source",
"write_metadata_gguf",
- "GgufTensor",
+ "GgufTensorInfo",
+ "gguf_split_paths",
+ "gguf_tensor_index",
"load_gguf_metadata",
"gguf_architecture",
- "iter_gguf_tensors",
"gguf_tensor_names",
]
diff --git a/python/freetoken/models/gguf/tensors.py b/python/freetoken/models/gguf/tensors.py
new file mode 100644
index 00000000..4cba1d08
--- /dev/null
+++ b/python/freetoken/models/gguf/tensors.py
@@ -0,0 +1,56 @@
+"""GGUF tensors as state-dict entries: float tensors in torch order, quantized ones as packed ggml rows grouped into the GGUF Linear's per-type segments."""
+
+from __future__ import annotations
+
+from typing import Iterator
+
+import numpy as np
+import torch
+
+from freetoken.layers.quantization.ggml_types import is_quantized, pad_rows, segment_param, segments
+
+from .reader import GgufTensorInfo
+
+_FLOAT_DTYPES = {0: torch.float32, 1: torch.float16, 30: torch.bfloat16}
+
+
+def float_tensor(info: GgufTensorInfo) -> torch.Tensor:
+ """A float tensor in torch order (ggml dims reversed)."""
+ return torch.from_numpy(np.array(info.data())).view(_FLOAT_DTYPES[info.ggml_type]).view(*reversed(info.ne))
+
+
+def packed_rows(info: GgufTensorInfo) -> torch.Tensor:
+ """Packed ggml rows ``[rows, row_bytes]`` of a quantized tensor."""
+ rows = int(np.prod(info.ne[1:]))
+ return torch.from_numpy(info.data()).view(rows, info.nbytes // rows)
+
+
+def matrix(info: GgufTensorInfo) -> torch.Tensor:
+ return packed_rows(info) if is_quantized(info.ggml_type) else float_tensor(info)
+
+
+def fused(prefix: str, infos: list[GgufTensorInfo], parts: list[torch.Tensor], k: int, pad_to: int = 0) -> Iterator[tuple[str, torch.Tensor]]:
+ """One fused module: float parts concatenated into ``weight``; with any quantized part, one matrix per ggml-type run (float runs as bf16 rows, quantized runs packed)."""
+ types = [i.ggml_type for i in infos]
+ pad = (-sum(p.shape[0] for p in parts)) % pad_to if pad_to else 0
+ if not any(is_quantized(t) for t in types):
+ rows = [p.to(torch.bfloat16) for p in parts]
+ if pad:
+ rows.append(torch.zeros(pad, *rows[0].shape[1:], dtype=torch.bfloat16))
+ yield f"{prefix}.weight", torch.cat(rows, dim=0)
+ return
+ segs = segments(tuple(types), tuple(p.shape[0] for p in parts))
+ for i, seg in enumerate(segs):
+ rows = parts[seg.first_part : seg.first_part + seg.num_parts]
+ zeros = pad if i == len(segs) - 1 else 0
+ if is_quantized(seg.ggml_type):
+ zeros += pad_rows(k)
+ dtype = torch.uint8
+ else:
+ rows = [r.to(torch.bfloat16) for r in rows]
+ dtype = torch.bfloat16
+ tail = [torch.zeros(zeros, rows[0].shape[1], dtype=dtype)] if zeros else []
+ yield f"{prefix}.{segment_param(i)}", torch.cat([*rows, *tail], dim=0)
+
+
+__all__ = ["float_tensor", "fused", "matrix", "packed_rows"]
diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py
index 6d5481c1..e6a5d4ac 100644
--- a/python/freetoken/models/gguf/tokenizer.py
+++ b/python/freetoken/models/gguf/tokenizer.py
@@ -13,7 +13,7 @@ from typing import Any
from .reader import gguf_architecture, load_gguf_metadata
# GGUF architecture -> transformers GGUF tokenizer-converter key.
-_TOKENIZER_ARCH = {"gemma4": "gemma4_text"}
+_TOKENIZER_ARCH = {"gemma4": "gemma4_text", "qwen4exp": "qwen3"}
def load_gguf_tokenizer(model_path: str):
diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.py
index 0fe4a4ff..15f866ca 100644
--- a/python/freetoken/models/qwen4_exp/__init__.py
+++ b/python/freetoken/models/qwen4_exp/__init__.py
@@ -10,6 +10,7 @@ Contracts shared across modules (do not rename):
"""
from .config import parse_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import Qwen4ExpForCausalLM, Qwen4ExpForConditionalGeneration
from .weight import (
ftw_side_files,
@@ -33,4 +34,7 @@ __all__ = [
"load_ple_table",
"parse_config",
"iter_expert_pieces",
+ "iter_gguf_weights",
+ "parse_gguf_config",
+ "to_gguf",
]
diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.py
index e4137862..b1e5f9aa 100644
--- a/python/freetoken/models/qwen4_exp/config.py
+++ b/python/freetoken/models/qwen4_exp/config.py
@@ -43,6 +43,8 @@ class Qwen4ExpArgs:
index_budget: int
index_ratio: int
image_token_id: int | None = None
+ # GGUF stores ssm_out with its value-head columns in llama.cpp's tiled order
+ gdn_out_heads_tiled: bool = False
@property
def index_topk_blocks(self) -> int:
diff --git a/python/freetoken/models/qwen4_exp/gdn.py b/python/freetoken/models/qwen4_exp/gdn.py
index ed49dd42..48c92a8e 100644
--- a/python/freetoken/models/qwen4_exp/gdn.py
+++ b/python/freetoken/models/qwen4_exp/gdn.py
@@ -32,9 +32,10 @@ class Qwen4ExpGatedDeltaNet(BaseOP):
def __init__(
self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,
conv_kernel_size, rms_norm_eps, layer_id, output_gate: str = "sigmoid",
- *, quant_config: QuantConfig | None = None, prefix: str = "",
+ *, quant_config: QuantConfig | None = None, prefix: str = "", out_heads_tiled: bool = False,
):
self.layer_id = layer_id
+ self._out_heads_tiled = out_heads_tiled
# The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as
# [V, K] while the LinearStatePool declares it [K, V]; these coincide (and the
# hybrid-radix snapshot scatter h[h_row]->slot is a plain copy) only when the two head
@@ -197,6 +198,10 @@ class Qwen4ExpGatedDeltaNet(BaseOP):
core_out = core_out.reshape(-1, self.head_v_dim)
z = z.reshape(-1, self.head_v_dim)
out = self.norm.forward(core_out, z).reshape(total, -1)
+ if self._out_heads_tiled:
+ # grouped value heads (k-head major) -> tiled (k-head minor), the column order of a GGUF ssm_out
+ r = self.num_v_heads // self.num_k_heads
+ out = out.view(total, self.num_k_heads, r, self.head_v_dim).transpose(1, 2).reshape(total, -1)
return self.out_proj.forward(out)
diff --git a/python/freetoken/models/qwen4_exp/gguf.py b/python/freetoken/models/qwen4_exp/gguf.py
new file mode 100644
index 00000000..7909324b
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/gguf.py
@@ -0,0 +1,264 @@
+"""Qwen3.8-Flash-Next from a llama.cpp ``qwen4exp`` GGUF: config from the GGUF metadata, tensor names, and the converter's transforms undone.
+
+The llama.cpp converter (conversion/qwen.py, conversion/qwen4exp.py) stores every zero-centred norm as ``w + 1``, ``A_log`` as ``-exp(A_log)``, the squeezed conv kernels, the indexer projection split into q and k, and the GDN value heads in tiled order (k-head minor). Row reorders are undone on the packed bytes; ``ssm_out`` keeps its tiled columns and the GDN layer permutes its input instead (``Qwen4ExpArgs.gdn_out_heads_tiled``).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Iterator
+
+import torch
+
+from freetoken.layers.quantization.ggml_types import is_quantized
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index
+from freetoken.models.gguf.tensors import float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+ from freetoken.models.gguf.config import GgufConfigShim
+
+_ARCH = "qwen4exp"
+
+# HF config fields the GGUF metadata does not carry (Qwen3.8-Flash-Next's own config.json values)
+_FAMILY_DEFAULTS = dict(
+ hidden_act="silu",
+ output_gate_type="sigmoid",
+ indexer_kv_heads=1,
+ ngram_vocab_size_base=20000000,
+ make_ngram_vocab_size_divisible_by=128,
+ split_ngram_parts=128,
+ norm_topk_prob=True,
+)
+
+_LAYER_LEAVES: dict[str, tuple[str, ...]] = {
+ "linear_attn.in_proj_qkv": ("attn_qkv",),
+ "linear_attn.in_proj_z": ("attn_gate",),
+ "linear_attn.in_proj_b": ("ssm_beta",),
+ "linear_attn.in_proj_a": ("ssm_alpha",),
+ "linear_attn.out_proj": ("ssm_out",),
+ "self_attn.q_proj": ("attn_q",),
+ "self_attn.k_proj": ("attn_k",),
+ "self_attn.v_proj": ("attn_v",),
+ "self_attn.o_proj": ("attn_output",),
+ "self_attn.indexer.index_qk_proj": ("indexer.q_proj", "indexer.k_proj"),
+ "mlp.gate": ("ffn_gate_inp",),
+ "mlp.shared_expert_gate": ("ffn_gate_inp_shexp",),
+ "mlp.shared_expert.gate_proj": ("ffn_gate_shexp",),
+ "mlp.shared_expert.up_proj": ("ffn_up_shexp",),
+ "mlp.shared_expert.down_proj": ("ffn_down_shexp",),
+ "mlp.experts.0.gate_proj": ("ffn_gate_exps",),
+ "mlp.experts.0.up_proj": ("ffn_up_exps",),
+ "mlp.experts.0.down_proj": ("ffn_down_exps",),
+ "attn_hyper_connection.input_mix_weight_down": ("hc_attn_down",),
+ "attn_hyper_connection.block_inject_weight": ("hc_attn_inject",),
+ "attn_hyper_connection.input_mix_weight_up": ("hc_attn_up",),
+ "mlp_hyper_connection.input_mix_weight_down": ("hc_ffn_down",),
+ "mlp_hyper_connection.block_inject_weight": ("hc_ffn_inject",),
+ "mlp_hyper_connection.input_mix_weight_up": ("hc_ffn_up",),
+ "ple.key_proj": ("ple_key",),
+ "ple.value_proj": ("ple_value",),
+}
+_TOP_LEAVES: dict[str, tuple[str, ...]] = {
+ "model.language_model.embed_tokens": ("token_embd",),
+ "lm_head": ("output",),
+ "model.language_model.hyper_connection_mixer.input_mix_weight_down": ("output_hc_down",),
+ "model.language_model.hyper_connection_mixer.input_mix_weight_up": ("output_hc_up",),
+}
+_LAYER_RE = re.compile(r"^model\.language_model\.layers\.(\d+)\.(.+)$")
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name in _TOP_LEAVES:
+ return _TOP_LEAVES[name]
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ return tuple(f"blk.{m.group(1)}.{leaf}" for leaf in _LAYER_LEAVES.get(m.group(2), ()))
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ md = shim.metadata
+
+ def kv(key: str):
+ return md[f"{_ARCH}.{key}"]
+
+ head_dim = int(kv("attention.key_length"))
+ sections = [int(s) for s in kv("rope.dimension_sections")][:3]
+ ratios = [int(r) for r in kv("attention.compress_ratios")]
+ n_v = int(kv("ssm.time_step_rank"))
+ text = SimpleNamespace(
+ **_FAMILY_DEFAULTS,
+ hidden_size=int(kv("embedding_length")),
+ # block_count also counts the MTP layers a GGUF may carry after the decoder
+ num_hidden_layers=int(kv("block_count")) - int(md.get(f"{_ARCH}.nextn_predict_layers", 0)),
+ num_attention_heads=int(kv("attention.head_count")),
+ num_key_value_heads=int(kv("attention.head_count_kv")),
+ head_dim=head_dim,
+ rope_parameters={
+ "rope_type": "default",
+ "rope_theta": float(kv("rope.freq_base")),
+ "partial_rotary_factor": int(kv("rope.dimension_count")) / head_dim,
+ "mrope_section": sections,
+ "mrope_interleaved": True,
+ },
+ max_position_embeddings=int(kv("context_length")),
+ full_attention_interval=int(kv("full_attention_interval")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ vocab_size=int(shim.vocab_size),
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ num_experts=int(kv("expert_count")),
+ num_experts_per_tok=int(kv("expert_used_count")),
+ moe_intermediate_size=int(kv("expert_feed_forward_length")),
+ shared_expert_intermediate_size=int(kv("expert_shared_feed_forward_length")),
+ linear_num_key_heads=int(kv("ssm.group_count")),
+ linear_num_value_heads=n_v,
+ linear_key_head_dim=int(kv("ssm.state_size")),
+ linear_value_head_dim=int(kv("ssm.inner_size")) // n_v,
+ linear_conv_kernel_dim=int(kv("ssm.conv_kernel")),
+ hc_count=int(kv("hyper_connection.count")),
+ hc_lowrank=int(kv("hyper_connection.low_rank")),
+ indexer_n_heads=int(kv("attention.indexer.head_count")),
+ indexer_head_dim=int(kv("attention.indexer.key_length")),
+ indexer_budget=int(kv("attention.indexer.top_k")),
+ indexer_compress_ratio=max(ratios),
+ ple_layer_ids=[int(i) + 1 for i in kv("ple.layers")],
+ # the GGUF key is one n-gram head's row; HF counts all (ngram_size - 1) * heads_per_ngram heads
+ ple_embed_dim=int(kv("embedding_length_per_layer_input")) * (int(kv("ple.ngram_size")) - 1) * int(kv("ple.heads_per_ngram")),
+ ple_conv_kernel_size=int(kv("ple.conv_kernel")),
+ ngram_size=int(kv("ple.ngram_size")),
+ heads_per_ngram=int(kv("ple.heads_per_ngram")),
+ eos_token_id=int(kv("ple.eos_token_id")),
+ )
+ image_token_id = md.get(f"{_ARCH}.ple.image_token_id")
+ hf_like = SimpleNamespace(
+ text_config=text,
+ model_type="qwen4_exp",
+ architectures=list(shim.architectures),
+ image_token_id=None if image_token_id is None else int(image_token_id),
+ vision_config=None,
+ )
+ config = parse_config(hf_like)
+ return replace(config, expert_quant="gguf", qwen4_args=replace(config.qwen4_args, gdn_out_heads_tiled=True))
+
+
+# ----------------------------------------------------------------------------------------------
+# Weights
+# ----------------------------------------------------------------------------------------------
+
+def _untile(t: torch.Tensor, num_k_heads: int, head_dim: int, dim: int = 0) -> torch.Tensor:
+ """Tiled value heads (k-head minor) back to grouped (k-head major) along ``dim``."""
+ n = t.shape[dim]
+ r = n // (num_k_heads * head_dim)
+ idx = torch.arange(n).view(r, num_k_heads, head_dim).transpose(0, 1).reshape(n)
+ return t.index_select(dim, idx)
+
+
+def iter_gguf_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+ include_vision: bool = True,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """The dense state dict from the GGUF; the routed experts load as offload banks (``freetoken.models.gguf.experts``)."""
+ if not include_non_moe:
+ return
+ from freetoken.utils import cached_load_hf_config
+
+ config = parse_gguf_config(cached_load_hf_config(model_path))
+ index = gguf_tensor_index(model_path)
+ md = cached_load_hf_config(model_path).metadata
+ g = config.linear_attention_group()
+ hidden = config.hidden_size
+
+ def t(name: str) -> GgufTensorInfo:
+ return index[name if name in index else f"{name}.weight"]
+
+ def norm(name: str) -> torch.Tensor:
+ return float_tensor(t(name)).float() - 1.0
+
+ def hc(prefix: str, src: str) -> Iterator[tuple[str, torch.Tensor]]:
+ down, inject, up = (t(f"{src}_{p}") for p in ("down", "inject", "up"))
+ yield from fused(f"{prefix}.input_mix_weight_down_block_inject", [down, inject], [matrix(down), matrix(inject)],
+ hidden * config.qwen4_args.hc_count, pad_to=16)
+ yield from fused(f"{prefix}.input_mix_weight_up", [up], [matrix(up)], config.qwen4_args.hc_lowrank)
+ yield f"{prefix}.hc_norm.weight", norm(f"{src}_norm")
+
+ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
+ for proj, k in (("down", hidden * config.qwen4_args.hc_count), ("up", config.qwen4_args.hc_lowrank)):
+ info = t(f"output_hc_{proj}")
+ yield from fused(f"model.hyper_connection_mixer.input_mix_weight_{proj}", [info], [matrix(info)], k)
+ yield "model.hyper_connection_mixer.hc_norm.weight", norm("output_hc_norm")
+
+ for layer in range(config.num_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ if config.is_linear_layer(layer):
+ la = f"{pre}.linear_attn"
+ nk, dk = g.num_key_heads, g.key_head_dim
+ qkv, gate = t(f"{blk}.attn_qkv"), t(f"{blk}.attn_gate")
+ qkv_rows = matrix(qkv)
+ v_start = 2 * nk * dk
+ qkv_rows = torch.cat([qkv_rows[:v_start], _untile(qkv_rows[v_start:], nk, g.value_head_dim)])
+ z_rows = _untile(matrix(gate), nk, g.value_head_dim)
+ beta_info, alpha_info = t(f"{blk}.ssm_beta"), t(f"{blk}.ssm_alpha")
+ beta, alpha = (_untile(matrix(i), nk, 1) for i in (beta_info, alpha_info))
+ if is_quantized(qkv.ggml_type) or is_quantized(gate.ggml_type):
+ yield from fused(f"{la}.in_proj_qkvz", [qkv, gate], [qkv_rows, z_rows], hidden)
+ yield from fused(f"{la}.in_proj_ba", [beta_info, alpha_info], [beta, alpha], hidden)
+ else:
+ yield from fused(f"{la}.in_proj", [qkv, gate, beta_info, alpha_info], [qkv_rows, z_rows, beta, alpha], hidden)
+ conv = float_tensor(t(f"{blk}.ssm_conv1d")).float()
+ conv = torch.cat([conv[:v_start], _untile(conv[v_start:], nk, g.value_head_dim)])
+ yield f"{la}.conv1d.weight", conv.unsqueeze(1)
+ yield f"{la}.dt_bias", _untile(float_tensor(t(f"{blk}.ssm_dt.bias")).float(), nk, 1)
+ yield f"{la}.A_log", _untile(torch.log(-float_tensor(t(f"{blk}.ssm_a")).float()), nk, 1)
+ yield f"{la}.norm.weight", float_tensor(t(f"{blk}.ssm_norm"))
+ yield from fused(f"{la}.out_proj", [t(f"{blk}.ssm_out")], [matrix(t(f"{blk}.ssm_out"))], g.num_value_heads * g.value_head_dim)
+ else:
+ sa = f"{pre}.self_attn"
+ parts = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
+ yield from fused(f"{sa}.qkv_proj", parts, [matrix(p) for p in parts], hidden)
+ out = t(f"{blk}.attn_output")
+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
+ yield f"{sa}.q_norm.weight", norm(f"{blk}.attn_q_norm")
+ yield f"{sa}.k_norm.weight", norm(f"{blk}.attn_k_norm")
+ iq, ik = t(f"{blk}.indexer.q_proj"), t(f"{blk}.indexer.k_proj")
+ yield from fused(f"{sa}.indexer.index_qk_proj", [iq, ik], [matrix(iq), matrix(ik)], hidden)
+ yield f"{sa}.indexer.q_layernorm.weight", norm(f"{blk}.indexer.q_norm")
+ yield f"{sa}.indexer.k_layernorm.weight", norm(f"{blk}.indexer.k_norm")
+
+ mlp = f"{pre}.mlp"
+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))
+ sg, su, sd = (t(f"{blk}.ffn_{p}_shexp") for p in ("gate", "up", "down"))
+ yield from fused(f"{mlp}.shared_expert.gate_up_proj", [sg, su], [matrix(sg), matrix(su)], hidden)
+ yield from fused(f"{mlp}.shared_expert.down_proj", [sd], [matrix(sd)], config.shared_expert_intermediate_size)
+ yield f"{mlp}.shared_expert_gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp_shexp")).reshape(1, -1)
+ yield from hc(f"{pre}.attn_hyper_connection", f"{blk}.hc_attn")
+ yield from hc(f"{pre}.mlp_hyper_connection", f"{blk}.hc_ffn")
+
+ if layer in config.qwen4_args.ple_layer_ids:
+ ple = f"{pre}.ple"
+ for proj in ("key", "value"):
+ info = t(f"{blk}.ple_{proj}")
+ yield from fused(f"{ple}.{proj}_proj", [info], [matrix(info)], config.qwen4_args.ple_embed_dim)
+ for n in ("key", "query", "conv"):
+ yield f"{ple}.norm_{n}.weight", norm(f"{blk}.ple_norm_{n}")
+ yield f"{ple}.conv1d.weight", float_tensor(t(f"{blk}.ple_conv1d")).float().unsqueeze(1)
+ yield f"{ple}.ple_embedding.layer_multipliers", torch.tensor([int(x) for x in md[f"{_ARCH}.ple.layer_multipliers"]], dtype=torch.int64)
+ yield f"{ple}.ple_embedding.ngram_heads_vocab_sizes", torch.tensor([int(x) for x in md[f"{_ARCH}.ple.head_vocab_sizes"]], dtype=torch.int64)
+ yield f"{ple}.ple_embedding.ngram_heads_offsets", torch.tensor([int(x) for x in md[f"{_ARCH}.ple.head_offsets"]], dtype=torch.int64)
+
+
+def ple_table_info(model_path: str) -> GgufTensorInfo:
+ return gguf_tensor_index(model_path)["per_layer_token_embd.weight"]
+
+
+__all__ = ["iter_gguf_weights", "parse_gguf_config", "ple_table_info", "to_gguf"]
diff --git a/python/freetoken/models/qwen4_exp/hc.py b/python/freetoken/models/qwen4_exp/hc.py
index f13808dc..69aaa583 100644
--- a/python/freetoken/models/qwen4_exp/hc.py
+++ b/python/freetoken/models/qwen4_exp/hc.py
@@ -98,6 +98,7 @@ class GatedResidual(BaseOP):
self.pad_size = (-(self.lowrank + self.hc_count)) % 16
self.input_mix_weight_down_block_inject = LinearReplicated(
width, self.lowrank + self.hc_count + self.pad_size, has_bias=False,
+ output_sizes=[self.lowrank, self.hc_count + self.pad_size],
quant_config=config.quant, prefix=f"{prefix}.input_mix_weight_down_block_inject",
)
else:
diff --git a/python/freetoken/models/qwen4_exp/model.py b/python/freetoken/models/qwen4_exp/model.py
index 258badac..4eb69165 100644
--- a/python/freetoken/models/qwen4_exp/model.py
+++ b/python/freetoken/models/qwen4_exp/model.py
@@ -52,6 +52,7 @@ def build_linear_mixer(config: ModelConfig, layer_id: int, prefix: str) -> BaseO
output_gate=g.output_gate,
quant_config=config.quant,
prefix=prefix,
+ out_heads_tiled=config.qwen4_args.gdn_out_heads_tiled,
)
@@ -92,6 +93,8 @@ class Qwen4ExpModel(BaseOP):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
+ quant_config=config.quant,
+ prefix=f"{prefix}.embed_tokens",
)
self.layers = OPList(
[
@@ -168,11 +171,8 @@ class Qwen4ExpForCausalLM(BaseLLMModel):
return 0
if engine_config.ple_backend == "disk":
- from freetoken.utils import download_hf_weight
-
from .ple_disk import DiskRowTable, resolve_row_source
- folder = download_hf_weight(engine_config.model_path)
# one WAIT node per captured graph: the flag protocol supports a single consume
assert len(ple_layers) == 1, "disk PLE backend expects exactly one PLE layer"
emb, args = ple_layers[0].ple_embedding, ple_layers[0].args
@@ -186,7 +186,7 @@ class Qwen4ExpForCausalLM(BaseLLMModel):
"image_token_id": args.image_token_id,
}
disk_table = DiskRowTable(
- resolve_row_source(folder),
+ resolve_row_source(engine_config.model_path),
constants,
max_graph_rows=max(256, engine_config.cuda_graph_max_bs or 0),
max_extend_tokens=engine_config.max_extend_tokens,
diff --git a/python/freetoken/models/qwen4_exp/ple_disk.py b/python/freetoken/models/qwen4_exp/ple_disk.py
index 69ee9fa9..bb71da2f 100644
--- a/python/freetoken/models/qwen4_exp/ple_disk.py
+++ b/python/freetoken/models/qwen4_exp/ple_disk.py
@@ -50,6 +50,9 @@ class PleRowSource:
row_bytes: int
row_stride: int
scale: float
+ # packed ggml rows (a GGUF table) of ``row_values`` values each; None for one fp8 byte per value
+ ggml_type: int | None = None
+ row_values: int | None = None
@property
def total_rows(self) -> int:
@@ -95,9 +98,27 @@ def source_from_safetensors(folder: str) -> PleRowSource:
return PleRowSource(paths, [f for f, _ in order], [b for _, b in order], rows, cols, cols, float(scale))
-def resolve_row_source(folder: str) -> PleRowSource:
+def source_from_gguf(model_path: str) -> PleRowSource:
+ """The GGUF ``per_layer_token_embd`` table in place: one extent of packed ggml rows."""
+ from freetoken.layers.quantization.ggml_types import row_bytes
+
+ from .gguf import ple_table_info
+
+ info = ple_table_info(model_path)
+ values, rows = info.ne
+ nbytes = row_bytes(info.ggml_type, values)
+ return PleRowSource([info.path], [0], [info.offset], rows, nbytes, nbytes, 1.0, info.ggml_type, values)
+
+
+def resolve_row_source(model_path: str) -> PleRowSource:
"""Pick the row source for a checkpoint; the seam where a repacked format would plug in."""
- return source_from_safetensors(folder)
+ from freetoken.models.gguf.reader import gguf_config_source
+
+ if (gguf_src := gguf_config_source(model_path)) is not None:
+ return source_from_gguf(gguf_src)
+ from freetoken.utils import download_hf_weight
+
+ return source_from_safetensors(download_hf_weight(model_path))
class DiskRowTable:
@@ -113,9 +134,14 @@ class DiskRowTable:
dtype: torch.dtype = torch.bfloat16,
) -> None:
from freetoken.kernel.row_store import PleStore
+ from freetoken.layers.quantization.ggml_types import float_dtype
self.num_rows = source.total_rows
- self.head_dim = source.row_bytes # fp8: one byte per element
+ self.head_dim = source.row_values or source.row_bytes # fp8: one byte per element
+ self._ggml_type = source.ggml_type
+ self._row_bytes = source.row_bytes
+ # float rows (fp8, or F32 / F16 / BF16 in a GGUF) are only cast; quantized ggml rows go through ggml
+ self._row_dtype = torch.float8_e4m3fn if source.ggml_type is None else float_dtype(source.ggml_type)
self.dtype = dtype
self.heads = int(hash_constants["num_ngram_heads"])
self.scale = source.scale
@@ -142,7 +168,7 @@ class DiskRowTable:
use_io_uring=os.getenv(_IO_URING_ENV, "1") != "0",
)
self._device = torch.device("cuda", torch.cuda.current_device())
- self._token_bytes = self.heads * self.head_dim
+ self._token_bytes = self.heads * self._row_bytes
# allocated up front: pinned alloc inside stream capture is illegal; one replay consumes it at a time
self._graph_pinned = alloc_pinned_tensor(max_graph_rows * self._token_bytes, dtype=torch.uint8)
self._graph_pinned.zero_() # padded decode lanes read whatever sits here
@@ -272,7 +298,13 @@ class DiskRowTable:
dev[:nbytes].copy_(pinned[:nbytes], non_blocking=True)
if not capturing:
self._eager_read[self._eager_slot].record(stream)
- values = dev[:nbytes].view(torch.float8_e4m3fn).to(self.dtype)
+ if self._row_dtype is not None:
+ # copy=True: rows already in self.dtype must not alias the staging buffer the next fill reuses
+ values = dev[:nbytes].view(self._row_dtype).to(self.dtype, copy=True)
+ else:
+ from freetoken.kernel import ggml
+
+ values = ggml.dequantize(dev[:nbytes].view(-1, self._row_bytes), self._ggml_type, self.head_dim, self.dtype)
if self.scale != 1.0:
values = values * self.scale
values = values.view(*row_ids.shape[:-1], -1)
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 678424ff..4b0cea25 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -213,6 +213,15 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
encoders=_QWEN_VL_ENCODERS,
packed_modules_mapping=_QWEN4_EXP_PACKED,
),
+ # llama.cpp qwen4exp GGUF (text only): the same model classes with every tensor in its packed ggml type
+ "Qwen4ExpGGUFForCausalLM": ModelSpec(
+ "freetoken.models.qwen4_exp",
+ "Qwen4ExpForCausalLM",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ packed_modules_mapping=_QWEN4_EXP_PACKED,
+ ),
# Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the
# qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the
# loader handles the compressed-tensors NVFP4 layout.
@@ -282,12 +291,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
checkpoint_segments=_GEMMA4_SEGMENTS,
packed_modules_mapping=_GEMMA4_PACKED,
),
- # GGUF (native Q4_0/Q6_K) gemma4: same model classes, GGUF config + weight loaders.
+ # llama.cpp gemma4 GGUF (text only): the same model classes with every tensor in its packed ggml type
"Gemma4GGUFForCausalLM": ModelSpec(
"freetoken.models.gemma4",
"Gemma4ForCausalLM",
parse_config="parse_gguf_config",
iter_weights="iter_gguf_weights",
+ checkpoint_segments=_GEMMA4_SEGMENTS,
+ packed_modules_mapping=_GEMMA4_PACKED,
),
"GptOssForCausalLM": ModelSpec(
"freetoken.models.gpt_oss",
@@ -342,16 +353,19 @@ def _load_attr(module_path: str, attr_name: str) -> Any:
def checkpoint_quant_config(model_path: str, hf_config: Any, spec: ModelSpec):
- """The checkpoint's QuantConfig under the family's naming, or None for GGUF, whose native-quant ops the shared parser does not model yet."""
- from freetoken.layers.quantization import NameMap, QuantConfig
+ """The checkpoint's QuantConfig under the family's naming.
+ A GGUF family gets the GGUF dialect over the file's tensor table, named through the ``to_gguf`` its module exports."""
+ from freetoken.layers.quantization import GGUFConfig, NameMap, QuantConfig
+
+ name_map = NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping)
if spec.parse_config == "parse_gguf_config":
- return None
- return QuantConfig.from_hf(
- hf_config,
- name_map=NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping),
- unquantized=spec.unquantized_modules,
- )
+ to_gguf = _load_attr(spec.module, "to_gguf")
+ from freetoken.models.gguf.reader import gguf_tensor_index
+
+ types = {name: info.ggml_type for name, info in gguf_tensor_index(hf_config.model_path).items()}
+ return GGUFConfig(types, to_gguf, name_map=name_map, unquantized=spec.unquantized_modules)
+ return QuantConfig.from_hf(hf_config, name_map=name_map, unquantized=spec.unquantized_modules)
def get_model_class(model_architecture: str, model_config: ModelConfig):
diff --git a/python/freetoken/models/weight.py b/python/freetoken/models/weight.py
index e3382b1b..b03266fb 100644
--- a/python/freetoken/models/weight.py
+++ b/python/freetoken/models/weight.py
@@ -262,28 +262,9 @@ def ftw_lacks_vision(model_path: str) -> bool:
return not any(name.startswith(VISION_KEY_PREFIXES) for name in ftw_tensor_names(model_path, "weight"))
-def load_q4_0_moe_expert_sources(
- model_path: str,
- model_config,
- *,
- dummy: bool = False,
- layer_sink=None,
-) -> dict:
- """Load (or fabricate, with ``dummy=True``) packed GGUF Q4_0 expert source banks.
- ``layer_sink`` (converter) streams each completed layer's banks; ignored for dummy."""
- _config, spec = _spec_for_model_path(model_path)
- if dummy:
- builder = _model_override(spec, "dummy_q4_0_expert_sources")
- assert builder is not None, "model defines no dummy_q4_0_expert_sources"
- return builder(model_config)
- loader = _load_attr(spec.module, "load_q4_0_expert_sources")
- return loader(model_path, model_config, layer_sink=layer_sink)
-
-
__all__ = [
"load_weight",
"load_vision_weight",
"ftw_lacks_vision",
- "load_q4_0_moe_expert_sources",
"iter_expert_tensors_parallel",
]
diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py
index 5521c34e..03ce4b75 100644
--- a/python/freetoken/moe/expert_banks.py
+++ b/python/freetoken/moe/expert_banks.py
@@ -2,8 +2,8 @@
The expert kernel (``QuantMethod.kernel``) owns the bank layout and the pack step; the
checkpoint side delivers pieces (``moe.expert_pieces``) and this module fills the pinned host
-banks from them (``build_expert_banks``). The GGUF q4_0 experts still
-use their own providers until they get a method.
+banks from them (``build_expert_banks``). GGUF experts keep their packed ggml rows
+(``models.gguf.experts``).
"""
from __future__ import annotations
@@ -19,7 +19,7 @@ from freetoken.layers.quantization import QuantKind
from freetoken.utils import init_logger
from .host_banks import alloc_layer_banks
-from .offload_cache import _BANK_BYTES_PER_EXPERT, _BANK_SCHEMAS
+from .offload_cache import _BANK_BYTES_PER_EXPERT
logger = init_logger(__name__)
@@ -50,6 +50,8 @@ class ExpertBanks:
layout: dict | None = None
# per-layer quant tag (e.g. the gate_up and down GGUF types), for layers whose banks match in dtype and row shape but not in quant; None = one quant
layer_tags: list | None = None
+ # per bank, per layer: bytes past an expert matrix's last row that the kernel reads and that must read as zeros (ggml MMQ); None = no such reads
+ row_tail_bytes: dict[str, list[int]] | None = None
def _dummy_fill(role: str, tensor: torch.Tensor) -> None:
@@ -161,46 +163,6 @@ def build_expert_banks(
_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader
-def _q4_0_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:
- if parallel:
- raise NotImplementedError(
- "parallel reader not implemented for q4_0: GGUF is a single packed file "
- "(not safetensors), so the common reader doesn't apply -- it needs a GGUF-native "
- "parallel reader (parse the tensor table, chunked O_DIRECT over the one file)"
- )
- from freetoken.models.weight import load_q4_0_moe_expert_sources
-
- # Native GGUF Q4_0 routed experts: packed block bytes streamed to the GPU and
- # dequantized inside the borrowed ggml MoE kernels (no bf16 expert copy). Banks are
- # per-layer HostBanks (pin-after-fill), so conversion streams each completed layer's
- # gate_up + down straight through the sink (dummy fabricates in one shot -> not streamed).
- sink = None if dummy else layer_sink
- sources = load_q4_0_moe_expert_sources(model_path, model_config, dummy=dummy, layer_sink=sink)
- return ExpertBanks(
- "q4_0", {name: sources[name] for name in _BANK_SCHEMAS["q4_0"]}, streamed=sink is not None
- )
-
-
-# expert formats that still load through their own provider (GGUF)
-_PROVIDERS = {
- "q4_0": _q4_0_banks,
-}
-
-
-def _legacy_expert_banks(model_path, model_config, device, dtype, dummy, parallel, workers, chunk, decode_target="gpu", layer_sink=None) -> ExpertBanks:
- expert_quant = model_config.expert_quant
- if expert_quant not in _PROVIDERS:
- raise ValueError(
- f"{expert_quant!r} experts load through their MoE quant method; "
- f"only {sorted(_PROVIDERS)} still have a format provider"
- )
- return _PROVIDERS[expert_quant](
- model_path, model_config, device, dtype, dummy,
- parallel=parallel, workers=workers, chunk=chunk, decode_target=decode_target,
- layer_sink=layer_sink,
- )
-
-
def _method_expert_banks(model_path, model_config, method, device, dummy, parallel, workers, chunk, layer_sink=None, resident=False) -> ExpertBanks:
from freetoken.moe.expert_pieces import iter_expert_pieces
@@ -258,7 +220,7 @@ def bank_bytes_estimate(model_config, method=None) -> int | None:
"""Estimated total expert-bank bytes of a raw checkpoint before loading it.
With a bound expert ``method`` the kernel's layout gives the exact host bytes; otherwise the
- format-tag table sizes the GGUF format. ``None`` for unknown formats or missing dims
+ format-tag table sizes the format. ``None`` for unknown formats or missing dims
(callers then skip the pre-load sizing)."""
layers = getattr(model_config, "num_moe_layers", None)
if method is not None and layers:
@@ -287,12 +249,10 @@ def load_expert_banks(
*,
method=None,
device: torch.device,
- dtype: torch.dtype,
dummy: bool = False,
parallel: bool | None = None,
workers: int = 8,
chunk: int = _PARALLEL_CHUNK,
- decode_target: str = "gpu",
layer_sink=None,
layer_residency: list[str] | None = None,
resident: bool = False,
@@ -315,8 +275,8 @@ def load_expert_banks(
streamable formats, so callers must check ``streamed`` rather than assume it fired.
``method`` (the bound expert quant method of the model's offload layers) selects
- the generic path: the family's pieces packed by the method's kernel. Without it only the
- GGUF q4_0 format loads, through its own provider.
+ the generic path: the family's pieces packed by the method's kernel. Without it only an
+ FTW checkpoint loads.
``layer_residency``: per-layer ``HostResidency`` labels applied at settle time -- explicitly on the FTW fast path, ambiently (``requested_residency``) in the slow-path providers.
Applied labels are echoed on ``ExpertBanks.layer_residency``; a loader that settles some other way leaves it ``None`` (CPU-layer decode still works on pinned banks, it just saves no pin quota).
@@ -370,9 +330,14 @@ def load_expert_banks(
from freetoken.moe.host_banks import requested_residency
def _build(par: bool) -> ExpertBanks:
- if method is not None:
- return _method_expert_banks(model_path, model_config, method, device, dummy, par, workers, chunk, layer_sink, resident)
- return _legacy_expert_banks(model_path, model_config, device, dtype, dummy, par, workers, chunk, decode_target, layer_sink)
+ if method is not None and method.kind is QuantKind.GGUF:
+ from freetoken.models.gguf.experts import load_gguf_expert_banks
+
+ # each layer keeps its own ggml types, so the banks come from the file, not from one method's layout
+ return load_gguf_expert_banks(model_path, model_config.num_moe_layers, dummy=dummy, device=device if resident else None)
+ if method is None:
+ raise ValueError("the experts of a raw checkpoint are packed by the offload layers' MoE quant method; pass method=")
+ return _method_expert_banks(model_path, model_config, method, device, dummy, par, workers, chunk, layer_sink, resident)
with requested_residency(layer_residency) as residency_plan:
try:
diff --git a/python/freetoken/moe/fused_q4_0.py b/python/freetoken/moe/fused_q4_0.py
deleted file mode 100644
index cdab82bf..00000000
--- a/python/freetoken/moe/fused_q4_0.py
+++ /dev/null
@@ -1,53 +0,0 @@
-"""Grouped expert GEMM over native GGUF Q4_0 banks (borrowed ggml MoE kernels).
-
-Ports vLLM/sglang's ``_fused_moe_gguf`` MMVQ path onto FreeToken's offload-cache
-interface: the experts are streamed to the GPU as packed Q4_0 block bytes and
-dequantized *inside* ``ggml_moe_a8_vec`` -- no bf16 expert copy is materialized. We
-use the MMVQ (vector) kernel for both prefill and decode: it consumes ``topk_ids``
-directly (no ``moe_align_block_size`` needed) and on small batches it is the right
-choice anyway. ``topk_ids`` already index the streamed cache slots (decode) or the
-materialized layer positions (prefill).
-"""
-
-from __future__ import annotations
-
-import torch
-
-from freetoken.layers.activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul
-from freetoken.models.gguf.dequant import GGML_Q4_0
-
-_ACT = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}
-
-
-def fused_experts_gguf_q4_0(
- hidden_states: torch.Tensor,
- gate_up_q: torch.Tensor, # [num_slots, 2I, H//32*18] uint8
- down_q: torch.Tensor, # [num_slots, H, I//32*18] uint8
- topk_weights: torch.Tensor,
- topk_ids: torch.Tensor,
- activation: str,
-) -> torch.Tensor:
- from freetoken.kernel.gguf import ggml_moe_a8_vec
-
- act_fn = _ACT.get(activation)
- if act_fn is None:
- raise ValueError(f"unsupported MoE activation {activation!r}")
-
- num_tokens = hidden_states.shape[0]
- n2 = gate_up_q.shape[1] # 2 * intermediate
- h = down_q.shape[1] # hidden
- top_k = topk_ids.shape[1]
- qt = int(GGML_Q4_0)
-
- # gate_up: [num_tokens*top_k, 2I] -> activation -> [num_tokens*top_k, I]
- gate_up = ggml_moe_a8_vec(hidden_states, gate_up_q, topk_ids, top_k, qt, n2, num_tokens)
- inter = act_fn(gate_up)
- # down: each of the num_tokens*top_k intermediate rows uses its own expert id.
- out = ggml_moe_a8_vec(inter, down_q, topk_ids, 1, qt, h, num_tokens * top_k)
- out = out.reshape(num_tokens, top_k, h) * topk_weights.reshape(num_tokens, top_k, 1).to(
- out.dtype
- )
- return out.sum(dim=1)
-
-
-__all__ = ["fused_experts_gguf_q4_0"]
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index 701524e0..a209f9a9 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -46,9 +46,8 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {
# Half the host/cache footprint of bf16; the grouped GEMM (kernel/triton/fp8_blockscale_moe)
# reads the routed fp8 rows directly and dequantizes in the K-loop (no bf16 materialization).
"fp8_block": ("gate_up", "gate_up_scale", "down", "down_scale"),
- # native GGUF Q4_0 experts: packed block bytes per output row, dequantized inside
- # the borrowed ggml MoE kernels. gate_up [L*E, 2I, H//32*18], down [L*E, H, I//32*18].
- "q4_0": ("gate_up", "down"),
+ # GGUF experts in their own ggml type per layer: gate_up [L*E, 2I, row_bytes] (gate rows, then up rows), down [L*E, H, row_bytes]; float experts keep their dtype
+ "gguf": ("gate_up", "down"),
# native ModelOpt rows for the Triton inline-dequant kernels: packed e2m1 codes +
# fp8-e4m3 per-16 block scales + per-output-row fp16 globals (w1/w3 carry distinct
# globals, and folding them into the e4m3 block scales would underflow)
@@ -93,7 +92,6 @@ _BANK_BYTES_PER_EXPERT = {
(2 * I // 128) * fp8_block_scale_pad(2 * I // 128, H // 128)
+ (H // 128) * fp8_block_scale_pad(H // 128, I // 128)
) * 2,
- "q4_0": lambda H, I: 2 * I * (H // 32) * 18 + H * (I // 32) * 18,
"nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2),
"mxfp4": lambda H, I: 2 * I * (H // 2 + H // 32 + 2) + H * (I // 2 + I // 32 + 2),
"ds_fp4": lambda H, I: 2 * I * (H // 2 + H // 32) + H * (I // 2 + I // 32),
@@ -182,13 +180,12 @@ class OffloadMoeCache:
# all layers = the plain --moe-strategy cpu case).
self.cpu_layer_ids: frozenset = frozenset()
# Bank row layouts, set by set_bank_sources: layouts[g][k] is (dtype, row shape) of bank k (layouts of two quant tags may be equal), layer_layout[l] is layer l's g.
- # With one (dtype, row shape) per bank a bank cache is typed [cache_size, *row]; with several it is a byte segment each pool views with its own layout.
+ # With one (dtype, row shape) per bank a bank cache is typed [cache_size, *row]; with several, or with pads, it is a byte segment each pool views with its own layout.
self.layouts: list[tuple[tuple[torch.dtype, tuple[int, ...]], ...]] = []
self.layer_layout: list[int] = [0] * self.num_layers
+ # _pad_bytes[g][k]: zeroed bytes after the last row of every pool and staging buffer of layout g in bank k, for kernels that read past it (16-aligned)
+ self._pad_bytes: list[list[int]] = []
self._raw_banks = False
- # Set by the engine when hybrid decode hands CPU routes to a kernel that reads slot 0 with weight 0 (no supports_inactive_slots).
- # With several quants a staging buffer can leave another quant's bytes in a pool's slot 0, which may decode to NaN, so the last prefill layer zeroes those rows.
- self.zero_inactive_route_slots = False
# num_experts floor + nvfp4_marlin slot cap, shared with the runtime-rebuild path.
self.validate_rebuild(self.cache_size, self.pool_sizes)
assert not self.prefill_overlap or self.cache_size >= 2 * self.num_experts, (
@@ -302,6 +299,8 @@ class OffloadMoeCache:
# _stage_views[g][b] are buffer b's per-bank [num_experts, ...] views for layout g; _inval[b] are the global slots it overwrites.
self._stage_bytes: list[int] = []
self._stage_views: list[list[tuple[torch.Tensor, ...] | None]] = []
+ # _stage_tails[g][b]: the pad after buffer b's rows in layout g
+ self._stage_tails: list[list[list[torch.Tensor]]] = []
self._inval: list[slice | torch.Tensor] = []
self.prefill_copy_stream: torch.cuda.Stream | None = None
self.prefill_begin_event: torch.cuda.Event | None = None
@@ -337,6 +336,12 @@ class OffloadMoeCache:
dtype, shape = self.layouts[layout][k]
return cache[offset : offset + nrows * row].view(dtype).view(nrows, *shape)
+ def _bank_bytes(self, k: int, pool_sizes: list[int]) -> int:
+ """Bytes of raw bank ``k``: each pool's slots and pad, plus the largest pad, so two staging buffers fit wherever two layers of slots do."""
+ layouts = [self._pool_layout(p) for p in range(len(pool_sizes))]
+ slots = sum(s * self._row_bytes[g][k] + self._pad_bytes[g][k] for g, s in zip(layouts, pool_sizes))
+ return slots + max(pads[k] for pads in self._pad_bytes)
+
def _stage_fits(self, buffers: int) -> bool:
return all(c.numel() * c.element_size() >= buffers * s for (_, c), s in zip(self.banks, self._stage_bytes))
@@ -356,14 +361,15 @@ class OffloadMoeCache:
# rows[k][p]: bytes of one slot of pool p in bank k; before the banks exist, one unit per slot with a stage of E
if self.banks:
rows = [[self._row_bytes[self._pool_layout(p)][k] for p in range(len(spans))] for k in range(len(self.banks))]
+ pads = [[self._pad_bytes[self._pool_layout(p)][k] for p in range(len(spans))] for k in range(len(self.banks))]
stage = self._stage_bytes
else:
- rows, stage = [[1] * len(spans)], [E]
+ rows, pads, stage = [[1] * len(spans)], [[0] * len(spans)], [E]
offsets = []
- for row in rows:
+ for row, pad in zip(rows, pads):
offsets.append([0])
- for s, r in zip(self.pool_sizes[:-1], row):
- offsets[-1].append(offsets[-1][-1] + s * r)
+ for s, r, d in zip(self.pool_sizes[:-1], row, pad):
+ offsets[-1].append(offsets[-1][-1] + s * r + d)
self._pool_offsets = offsets
# local slots below this index overlap the prefill staging area in some bank
self._pool_stable_from = [
@@ -371,10 +377,8 @@ class OffloadMoeCache:
for p, s in enumerate(self.pool_sizes)
]
self._inval = []
- masks = []
for b in range(2):
mask = torch.zeros(self.cache_size, dtype=torch.bool)
- masks.append(mask)
for k in range(len(rows)):
lo, hi = b * stage[k], (b + 1) * stage[k]
for p, (base, s) in enumerate(spans):
@@ -387,9 +391,8 @@ class OffloadMoeCache:
else:
self._inval.append(idx.to(self.device))
self._stage_dirty = [False, False]
- self._stage_slot0_rows = [[], []]
if not self.banks:
- self._pool_bank_views, self._stage_views = [], []
+ self._pool_bank_views, self._stage_views, self._stage_tails = [], [], []
return
self._pool_bank_views = [
tuple(self._bank_rows(k, self._pool_layout(p), offsets[k][p], s) for k in range(len(self.banks)))
@@ -402,11 +405,10 @@ class OffloadMoeCache:
]
for g in range(len(self.layouts))
]
- if len(self.layouts) > 1:
- self._stage_slot0_rows = [
- [self._bank_rows(k, self._pool_layout(p), offsets[k][p], 1) for p, base in enumerate(bases) if masks[b][base] for k in range(len(self.banks))]
- for b in range(2)
- ]
+ self._stage_tails = [
+ [[self.banks[k][1][b * stage[k] + E * row[k] :][: pad[k]] for k in range(len(self.banks)) if pad[k] and self._stage_fits(b + 1)] for b in range(2)]
+ for row, pad in zip(self._row_bytes, self._pad_bytes)
+ ]
def stage_views(self, layer_id: int, buffer_id: int) -> tuple[torch.Tensor, ...]:
"""Per-bank ``[num_experts, ...]`` views of prefill staging buffer ``buffer_id`` in ``layer_id``'s layout; row == expert id."""
@@ -432,6 +434,7 @@ class OffloadMoeCache:
sources: dict[str, list[torch.Tensor]],
layer_residency: list[str] | None = None,
layer_tags: list | None = None,
+ row_tail_bytes: dict[str, list[int]] | None = None,
) -> None:
"""Attach the host (CPU pinned) expert source banks and allocate a GPU slot
cache per bank, following the format's bank schema.
@@ -447,6 +450,8 @@ class OffloadMoeCache:
Non-pinned (LOCKED/PAGEABLE) layers have no device address: they must already be routed to the CPU executor (``cpu_layer_ids``, set BEFORE this call), the copy plan skips their rows, and their only movement is ``copy_missing``'s whole-layer pageable prefill branch -- which is why prefill overlap is incompatible with them.
``layer_tags`` names each layer's quant (see :func:`bank_layouts`); layers of different tags never share a pool.
+
+ ``row_tail_bytes`` gives, per bank and layer, how many bytes past the last row the kernel reads; every pool and staging buffer is then followed by that many zero bytes.
"""
from freetoken.moe.legacy_format import canonical_role
from freetoken.moe.host_banks import HostResidency
@@ -498,12 +503,22 @@ class OffloadMoeCache:
for p in range(len(self.pool_sizes)):
mixed = {self.layer_layout[l] for l, q in enumerate(self.pool_of_layer) if q == p}
assert len(mixed) == 1, f"slot pool {p} holds layers of {len(mixed)} layouts; a pool needs one"
- self._raw_banks = len(set(self.layouts)) > 1
+ self._pad_bytes = [[0] * len(self.bank_schema) for _ in self.layouts]
+ if row_tail_bytes is not None:
+ tails = {canonical_role(name): per_layer for name, per_layer in row_tail_bytes.items()}
+ for k, name in enumerate(self.bank_schema):
+ for layer_id, tail in enumerate(tails[canonical_role(name)]):
+ g = self.layer_layout[layer_id]
+ self._pad_bytes[g][k] = max(self._pad_bytes[g][k], -(-tail // 16) * 16)
+ self._raw_banks = len(set(self.layouts)) > 1 or any(map(any, self._pad_bytes))
self._row_bytes = [[math.prod(shape) * dtype.itemsize for dtype, shape in layout] for layout in self.layouts]
if self._raw_banks:
# every pool view and staging buffer starts on a row boundary of its own layout
assert all(r % 16 == 0 for rows in self._row_bytes for r in rows), self._row_bytes
- self._stage_bytes = [self.num_experts * max(rows[k] for rows in self._row_bytes) for k in range(len(self.bank_schema))]
+ self._stage_bytes = [
+ self.num_experts * max(rows[k] for rows in self._row_bytes) + max(pads[k] for pads in self._pad_bytes)
+ for k in range(len(self.bank_schema))
+ ]
self._alloc_banks()
if not self._stage_fits(1):
raise ValueError(f"MoE cache of {self.cache_size} slots cannot stage one whole layer of experts")
@@ -518,8 +533,7 @@ class OffloadMoeCache:
def _alloc_banks(self) -> None:
for k, name in enumerate(self.bank_schema):
if self._raw_banks:
- nbytes = sum(s * self._row_bytes[self._pool_layout(p)][k] for p, s in enumerate(self.pool_sizes))
- self.bank_caches[name] = torch.zeros(nbytes, dtype=torch.uint8, device=self.device)
+ self.bank_caches[name] = torch.zeros(self._bank_bytes(k, self.pool_sizes), dtype=torch.uint8, device=self.device)
else:
dtype, shape = self.layouts[0][k]
self.bank_caches[name] = torch.zeros((self.cache_size, *shape), dtype=dtype, device=self.device)
@@ -634,7 +648,7 @@ class OffloadMoeCache:
)
if self._raw_banks and pool_sizes is not None:
for k, stage in enumerate(self._stage_bytes):
- if sum(s * self._row_bytes[self._pool_layout(p)][k] for p, s in enumerate(pool_sizes)) < stage:
+ if self._bank_bytes(k, pool_sizes) < stage:
raise ValueError(f"moe_cache_size={cache_size} cannot stage one whole layer of experts")
def rebuild(self, cache_size: int, pool_sizes: list[int] | None = None) -> None:
@@ -663,7 +677,7 @@ class OffloadMoeCache:
self.bank_caches = {}
self._pool_bank_views = []
self._stage_views = []
- self._stage_slot0_rows = [[], []]
+ self._stage_tails = []
self.cache_size = cache_size
self.pool_sizes = sizes
if self.device.type == "cuda":
@@ -796,7 +810,8 @@ class OffloadMoeCache:
)
self._prefill_hit_num = torch.zeros((1,), dtype=torch.int64, device=self.device)
- def _invalidate_prefill_buffer(self, buffer_id: int) -> None:
+ def _invalidate_prefill_buffer(self, buffer_id: int, layer_id: int) -> None:
+ """Take buffer ``buffer_id``'s slots for staging ``layer_id``."""
slots = self._inval[buffer_id]
old_ids = self.id_of_slot[slots]
self.slot_for_id.view(-1)[old_ids[old_ids >= 0].long()] = -1
@@ -805,6 +820,9 @@ class OffloadMoeCache:
# ensure_experts evicts them first.
self.usage[slots] = 0
self._stage_dirty[buffer_id] = True
+ # the pad may hold rows of a wider layout or of a pool
+ for tail in self._stage_tails[self.layer_layout[layer_id]][buffer_id]:
+ tail.zero_()
def begin_prefill(self) -> None:
if not self.prefill_overlap:
@@ -846,7 +864,7 @@ class OffloadMoeCache:
)
def copy() -> None:
- self._invalidate_prefill_buffer(buffer_id)
+ self._invalidate_prefill_buffer(buffer_id, layer_id)
for (per_layer, _), view in zip(self.banks, self.stage_views(layer_id, buffer_id)):
view.copy_(per_layer[layer_id], non_blocking=True)
@@ -954,7 +972,7 @@ class OffloadMoeCache:
with torch.cuda.stream(self.prefill_copy_stream):
if self._prefill_buffer_has_release_event[buffer_id]:
self.prefill_copy_stream.wait_event(self.prefill_release_events[buffer_id])
- self._invalidate_prefill_buffer(buffer_id)
+ self._invalidate_prefill_buffer(buffer_id, layer_id)
if miss.size:
run_starts = np.concatenate(([0], np.nonzero(np.diff(miss) != 1)[0] + 1))
starts = miss[run_starts]
@@ -997,12 +1015,13 @@ class OffloadMoeCache:
return self.stage_views(layer_id, buffer_id)
def release_prefill_layer(self, layer_id: int) -> None:
- if self.zero_inactive_route_slots and layer_id == self.num_layers - 1:
+ if layer_id == self.num_layers - 1:
# the staged slots were invalidated and nothing registers them within the forward, so their bytes are free to clear
for b in range(2):
if self._stage_dirty[b]:
- for row in self._stage_slot0_rows[b]:
- row.zero_()
+ # staged rows left there may be shifted or of another quant, and kernels still read them: MMQ past a slot's last row, weight-0 routes on slot 0
+ for (_, cache), stage in zip(self.banks, self._stage_bytes):
+ cache.view(-1).view(torch.uint8)[b * stage : (b + 1) * stage].zero_()
self._stage_dirty[b] = False
if not self.prefill_overlap:
return
diff --git a/python/freetoken/moe/offload_kernels.py b/python/freetoken/moe/offload_kernels.py
index 1d863a2f..8c4ad8f3 100644
--- a/python/freetoken/moe/offload_kernels.py
+++ b/python/freetoken/moe/offload_kernels.py
@@ -108,7 +108,7 @@ def materialize_layer(cache, layer_id: int) -> None:
return
# staging buffer 0 may overlap several pools, so it only holds the layer for this forward and registers nothing
E = cache.num_experts
- cache._invalidate_prefill_buffer(0)
+ cache._invalidate_prefill_buffer(0, layer_id)
rows = torch.arange(E, dtype=torch.int32, device=cache.device)
cache.evict_slots[:E].copy_(rows)
cache.src_indices[:E].copy_(rows)
diff --git a/python/freetoken/third_party/__init__.py b/python/freetoken/third_party/__init__.py
new file mode 100644
index 00000000..633cc8a7
--- /dev/null
+++ b/python/freetoken/third_party/__init__.py
@@ -0,0 +1 @@
+"""Vendored third-party Python code; each package carries its LICENSE and VERSION."""
diff --git a/python/freetoken/third_party/gguf/LICENSE b/python/freetoken/third_party/gguf/LICENSE
new file mode 100644
index 00000000..76f67efd
--- /dev/null
+++ b/python/freetoken/third_party/gguf/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Georgi Gerganov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/python/freetoken/third_party/gguf/VERSION b/python/freetoken/third_party/gguf/VERSION
new file mode 100644
index 00000000..f3ec79b9
--- /dev/null
+++ b/python/freetoken/third_party/gguf/VERSION
@@ -0,0 +1,10 @@
+llama.cpp 8c2134a8755d8a9792af513dab20282affe903e4 (2026-10-14), the same pin as kernel/csrc/ggml
+
+Only the gguf-py modules that the GGUF reader and the ggml type tables need: gguf-py/gguf/{constants,gguf_reader,quants,lazy}.py.
+The PyPI gguf release lags the ggml types in this tree (gguf 0.19.0 cannot read Q2_0 tensors), so FreeToken reads GGUF files with this copy.
+
+Local changes, kept in freetoken.patch (already applied):
+- __init__.py: imports only the vendored modules (upstream also pulls in the writer, vocab and metadata modules and their dependencies).
+- gguf_reader.py: imports constants relatively, so the installed gguf package is never picked up.
+
+To update: copy the same files from the llama.cpp commit that kernel/csrc/ggml pins, apply freetoken.patch, and update this file.
diff --git a/python/freetoken/third_party/gguf/__init__.py b/python/freetoken/third_party/gguf/__init__.py
new file mode 100644
index 00000000..f4c74ef9
--- /dev/null
+++ b/python/freetoken/third_party/gguf/__init__.py
@@ -0,0 +1,4 @@
+from .constants import *
+from .lazy import *
+from .gguf_reader import *
+from .quants import *
diff --git a/python/freetoken/third_party/gguf/constants.py b/python/freetoken/third_party/gguf/constants.py
new file mode 100644
index 00000000..c31ab4a4
--- /dev/null
+++ b/python/freetoken/third_party/gguf/constants.py
@@ -0,0 +1,6332 @@
+from __future__ import annotations
+
+from enum import Enum, IntEnum, auto
+from typing import Any
+
+#
+# constants
+#
+
+GGUF_MAGIC = 0x46554747 # "GGUF"
+GGUF_VERSION = 3
+GGUF_DEFAULT_ALIGNMENT = 32
+GGML_QUANT_VERSION = 2 # GGML_QNT_VERSION from ggml.h
+GGML_MAX_DIMS = 4 # GGML_MAX_DIMS from ggml.h
+
+#
+# metadata keys
+#
+
+
+class Keys:
+ class General:
+ TYPE = "general.type"
+ ARCHITECTURE = "general.architecture"
+ QUANTIZATION_VERSION = "general.quantization_version"
+ ALIGNMENT = "general.alignment"
+ FILE_TYPE = "general.file_type"
+
+ # Per-tensor extra options (tensor name array + parallel option arrays, e.g. prec_a4).
+ TENSOR_EXTRA_NAME = "general.tensor_extra.name"
+ TENSOR_EXTRA_PREC_A4 = "general.tensor_extra.prec_a4"
+
+ # Recommended Sampler Parameters
+ SAMPLING_SEQUENCE = "general.sampling.sequence"
+ SAMPLING_TOP_K = "general.sampling.top_k"
+ SAMPLING_TOP_P = "general.sampling.top_p"
+ SAMPLING_MIN_P = "general.sampling.min_p"
+ SAMPLING_XTC_PROBABILITY = "general.sampling.xtc_probability"
+ SAMPLING_XTC_THRESHOLD = "general.sampling.xtc_threshold"
+ SAMPLING_TEMP = "general.sampling.temp"
+ SAMPLING_PENALTY_LAST_N = "general.sampling.penalty_last_n"
+ SAMPLING_PENALTY_REPEAT = "general.sampling.penalty_repeat"
+ SAMPLING_MIROSTAT = "general.sampling.mirostat"
+ SAMPLING_MIROSTAT_TAU = "general.sampling.mirostat_tau"
+ SAMPLING_MIROSTAT_ETA = "general.sampling.mirostat_eta"
+
+ # Authorship Metadata
+ NAME = "general.name"
+ AUTHOR = "general.author"
+ VERSION = "general.version"
+ ORGANIZATION = "general.organization"
+
+ FINETUNE = "general.finetune"
+ BASENAME = "general.basename"
+
+ DESCRIPTION = "general.description"
+ QUANTIZED_BY = "general.quantized_by"
+
+ SIZE_LABEL = "general.size_label"
+
+ # Licensing details
+ LICENSE = "general.license"
+ LICENSE_NAME = "general.license.name"
+ LICENSE_LINK = "general.license.link"
+
+ # Typically represents the converted GGUF repo (Unless native)
+ URL = "general.url" # Model Website/Paper
+ DOI = "general.doi"
+ UUID = "general.uuid"
+ REPO_URL = "general.repo_url" # Model Source Repository (git/svn/etc...)
+
+ # Model Source during conversion
+ SOURCE_URL = "general.source.url" # Model Website/Paper
+ SOURCE_DOI = "general.source.doi"
+ SOURCE_UUID = "general.source.uuid"
+ SOURCE_REPO_URL = "general.source.repo_url" # Model Source Repository (git/svn/etc...)
+
+ # Base Model Source. There can be more than one source if it's a merged
+ # model like with 'Mistral-7B-Merge-14-v0.1'. This will assist in
+ # tracing linage of models as it is finetuned or merged over time.
+ BASE_MODEL_COUNT = "general.base_model.count"
+ BASE_MODEL_NAME = "general.base_model.{id}.name"
+ BASE_MODEL_AUTHOR = "general.base_model.{id}.author"
+ BASE_MODEL_VERSION = "general.base_model.{id}.version"
+ BASE_MODEL_ORGANIZATION = "general.base_model.{id}.organization"
+ BASE_MODEL_DESCRIPTION = "general.base_model.{id}.description"
+ BASE_MODEL_URL = "general.base_model.{id}.url" # Model Website/Paper
+ BASE_MODEL_DOI = "general.base_model.{id}.doi"
+ BASE_MODEL_UUID = "general.base_model.{id}.uuid"
+ BASE_MODEL_REPO_URL = "general.base_model.{id}.repo_url" # Model Source Repository (git/svn/etc...)
+
+ # Dataset Source
+ DATASET_COUNT = "general.dataset.count"
+ DATASET_NAME = "general.dataset.{id}.name"
+ DATASET_AUTHOR = "general.dataset.{id}.author"
+ DATASET_VERSION = "general.dataset.{id}.version"
+ DATASET_ORGANIZATION = "general.dataset.{id}.organization"
+ DATASET_DESCRIPTION = "general.dataset.{id}.description"
+ DATASET_URL = "general.dataset.{id}.url" # Model Website/Paper
+ DATASET_DOI = "general.dataset.{id}.doi"
+ DATASET_UUID = "general.dataset.{id}.uuid"
+ DATASET_REPO_URL = "general.dataset.{id}.repo_url" # Model Source Repository (git/svn/etc...)
+
+ # Array based KV stores
+ TAGS = "general.tags"
+ LANGUAGES = "general.languages"
+
+ class LLM:
+ VOCAB_SIZE = "{arch}.vocab_size"
+ CONTEXT_LENGTH = "{arch}.context_length"
+ EMBEDDING_LENGTH = "{arch}.embedding_length"
+ EMBEDDING_LENGTH_OUT = "{arch}.embedding_length_out"
+ FEATURES_LENGTH = "{arch}.features_length"
+ BLOCK_COUNT = "{arch}.block_count"
+ LEADING_DENSE_BLOCK_COUNT = "{arch}.leading_dense_block_count"
+ FEED_FORWARD_LENGTH = "{arch}.feed_forward_length"
+ EXPERT_FEED_FORWARD_LENGTH = "{arch}.expert_feed_forward_length"
+ EXPERT_SHARED_FEED_FORWARD_LENGTH = "{arch}.expert_shared_feed_forward_length"
+ EXPERT_CHUNK_FEED_FORWARD_LENGTH = "{arch}.expert_chunk_feed_forward_length"
+ USE_PARALLEL_RESIDUAL = "{arch}.use_parallel_residual"
+ TENSOR_DATA_LAYOUT = "{arch}.tensor_data_layout"
+ EXPERT_COUNT = "{arch}.expert_count"
+ EXPERT_USED_COUNT = "{arch}.expert_used_count"
+ EXPERT_SHARED_COUNT = "{arch}.expert_shared_count"
+ EXPERT_GROUP_COUNT = "{arch}.expert_group_count"
+ EXPERT_GROUP_USED_COUNT = "{arch}.expert_group_used_count"
+ EXPERT_WEIGHTS_SCALE = "{arch}.expert_weights_scale"
+ EXPERT_WEIGHTS_NORM = "{arch}.expert_weights_norm"
+ EXPERT_GATING_FUNC = "{arch}.expert_gating_func"
+ EXPERT_GROUP_SCALE = "{arch}.expert_group_scale"
+ EXPERT_LATENT_LENGTH = "{arch}.expert_latent_length"
+ EXPERTS_PER_GROUP = "{arch}.experts_per_group"
+ MOE_EVERY_N_LAYERS = "{arch}.moe_every_n_layers"
+ MOE_LATENT_SIZE = "{arch}.moe_latent_size"
+ NEXTN_PREDICT_LAYERS = "{arch}.nextn_predict_layers"
+ NUM_DEEPSTACK_LAYERS = "{arch}.n_deepstack_layers"
+ DEEPSTACK_MAPPING = "{arch}.deepstack_mapping"
+ POOLING_TYPE = "{arch}.pooling_type"
+ LOGIT_SCALE = "{arch}.logit_scale"
+ DECODER_START_TOKEN_ID = "{arch}.decoder_start_token_id"
+ DECODER_BLOCK_COUNT = "{arch}.decoder_block_count"
+ ATTN_LOGIT_SOFTCAPPING = "{arch}.attn_logit_softcapping"
+ ROUTER_LOGIT_SOFTCAPPING = "{arch}.router_logit_softcapping"
+ FINAL_LOGIT_SOFTCAPPING = "{arch}.final_logit_softcapping"
+ SWIN_NORM = "{arch}.swin_norm"
+ RESCALE_EVERY_N_LAYERS = "{arch}.rescale_every_n_layers"
+ TIME_MIX_EXTRA_DIM = "{arch}.time_mix_extra_dim"
+ TIME_DECAY_EXTRA_DIM = "{arch}.time_decay_extra_dim"
+ RESIDUAL_SCALE = "{arch}.residual_scale"
+ EMBEDDING_SCALE = "{arch}.embedding_scale"
+ TOKEN_SHIFT_COUNT = "{arch}.token_shift_count"
+ INTERLEAVE_MOE_LAYER_STEP = "{arch}.interleave_moe_layer_step"
+ FULL_ATTENTION_INTERVAL = "{arch}.full_attention_interval"
+ NUM_LOOPS = "{arch}.num_loops"
+ SKIP_LOOP_FINAL_NORM = "{arch}.skip_loop_final_norm"
+ HASH_LAYER_COUNT = "{arch}.hash_layer_count"
+ ACTIVATION_SPARSITY_SCALE = "{arch}.activation_sparsity_scale"
+ ALTUP_ACTIVE_IDX = "{arch}.altup.active_idx"
+ ALTUP_NUM_INPUTS = "{arch}.altup.num_inputs"
+ EMBD_LENGTH_PER_LAYER_INP = "{arch}.embedding_length_per_layer_input"
+ SWIGLU_CLAMP_EXP = "{arch}.swiglu_clamp_exp"
+ SWIGLU_CLAMP_SHEXP = "{arch}.swiglu_clamp_shexp"
+ HIDDEN_ACT = "{arch}.hidden_activation"
+ DENSE_FEAT_IN_SIZE = "{arch}.{dense}_feat_in"
+ DENSE_FEAT_OUT_SIZE = "{arch}.{dense}_feat_out"
+ TARGET_LAYERS = "{arch}.target_layers"
+ TARGET_HIDDEN_SIZE = "{arch}.target_hidden_size"
+ BLOCK_SIZE = "{arch}.block_size"
+ CONV_KERNEL_SIZE = "{arch}.conv_kernel_size"
+ CONV_GROUP_SIZE = "{arch}.conv_group_size"
+ SELECTOR_RANK = "{arch}.selector_rank"
+ SELECTOR_TOP_K = "{arch}.selector_top_k"
+ SAMPLE_FROM_ANCHOR = "{arch}.sample_from_anchor"
+ HAS_CONFIDENCE_HEAD = "{arch}.has_confidence_head"
+ NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual"
+ NORM_BEFORE_FC = "{arch}.norm_before_fc"
+
+ class Adapters:
+ COUNT = "{arch}.adapters.count"
+ TOKEN_IDS_ACTIVATE = "{arch}.adapters.token_ids_activate"
+ TOKEN_IDS_SUBSTITUTE = "{arch}.adapters.token_ids_substitute"
+ LORA_RANK = "{arch}.adapters.lora_rank"
+ ROUTER_GAIN = "{arch}.adapters.router_gain"
+
+ class Attention:
+ HEAD_COUNT = "{arch}.attention.head_count"
+ HEAD_COUNT_KV = "{arch}.attention.head_count_kv"
+ MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias"
+ CLAMP_KQV = "{arch}.attention.clamp_kqv"
+ KEY_LENGTH = "{arch}.attention.key_length"
+ VALUE_LENGTH = "{arch}.attention.value_length"
+ LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon"
+ LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon"
+ GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon"
+ GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups"
+ CAUSAL = "{arch}.attention.causal"
+ Q_LORA_RANK = "{arch}.attention.q_lora_rank"
+ KV_LORA_RANK = "{arch}.attention.kv_lora_rank"
+ DECAY_LORA_RANK = "{arch}.attention.decay_lora_rank"
+ ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank"
+ VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank"
+ GATE_LORA_RANK = "{arch}.attention.gate_lora_rank"
+ REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count"
+ SLIDING_WINDOW = "{arch}.attention.sliding_window"
+ SCALE = "{arch}.attention.scale"
+ OUTPUT_GROUP_COUNT = "{arch}.attention.output_group_count"
+ OUTPUT_LORA_RANK = "{arch}.attention.output_lora_rank"
+ OUTPUT_SCALE = "{arch}.attention.output_scale"
+ VALUE_SCALE = "{arch}.attention.value_scale"
+ COMPRESS_RATIOS = "{arch}.attention.compress_ratios"
+ COMPRESS_ROPE_FREQ_BASE = "{arch}.attention.compress_rope_freq_base"
+ TEMPERATURE_LENGTH = "{arch}.attention.temperature_length"
+ KEY_LENGTH_MLA = "{arch}.attention.key_length_mla"
+ VALUE_LENGTH_MLA = "{arch}.attention.value_length_mla"
+ KEY_LENGTH_SWA = "{arch}.attention.key_length_swa"
+ VALUE_LENGTH_SWA = "{arch}.attention.value_length_swa"
+ KEY_LENGTH_MLA_SWA = "{arch}.attention.key_length_mla_swa"
+ VALUE_LENGTH_MLA_SWA = "{arch}.attention.value_length_mla_swa"
+ KV_LORA_RANK_SWA = "{arch}.attention.kv_lora_rank_swa"
+ SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers"
+ SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern"
+ RECURRENT_LAYERS = "{arch}.attention.recurrent_layers"
+ TEMPERATURE_SCALE = "{arch}.attention.temperature_scale"
+ ROPE_PATTERN = "{arch}.attention.rope_pattern"
+ VALUE_EXPERT_COUNT = "{arch}.attention.value_expert_count"
+ VALUE_EXPERT_USED_COUNT = "{arch}.attention.value_expert_used_count"
+
+ class Indexer:
+ HEAD_COUNT = "{arch}.attention.indexer.head_count"
+ KEY_LENGTH = "{arch}.attention.indexer.key_length"
+ TOP_K = "{arch}.attention.indexer.top_k"
+ BLOCK_SIZE = "{arch}.attention.indexer.block_size" # MSA
+ LOCAL_BLOCKS = "{arch}.attention.indexer.local_blocks" # MSA
+ TYPES = "{arch}.attention.indexer.types"
+ KPOOL = "{arch}.attention.indexer.kpool" # GLM5-Next
+ KPOOL_SELECT_TAIL = "{arch}.attention.indexer.kpool_select_tail" # GLM5-Next
+
+ class HyperConnection:
+ COUNT = "{arch}.hyper_connection.count"
+ SINKHORN_ITERATIONS = "{arch}.hyper_connection.sinkhorn_iterations"
+ EPSILON = "{arch}.hyper_connection.epsilon"
+ # scale of the post gate (DeepSeek-V4 hardcodes 2.0)
+ MAGNITUDE = "{arch}.hyper_connection.magnitude"
+ # absent means the mix projection is full rank (DeepSeek-V4 behaviour)
+ LOW_RANK = "{arch}.hyper_connection.low_rank"
+
+ class PerLayerEmbedding:
+ LAYERS = "{arch}.ple.layers"
+ NGRAM_SIZE = "{arch}.ple.ngram_size"
+ HEADS_PER_NGRAM = "{arch}.ple.heads_per_ngram"
+ CONV_KERNEL = "{arch}.ple.conv_kernel"
+ LAYER_MULTIPLIERS = "{arch}.ple.layer_multipliers"
+ HEAD_OFFSETS = "{arch}.ple.head_offsets"
+ HEAD_VOCAB_SIZES = "{arch}.ple.head_vocab_sizes"
+ EOS_TOKEN_ID = "{arch}.ple.eos_token_id"
+ IMAGE_TOKEN_ID = "{arch}.ple.image_token_id"
+
+ class Rope:
+ DIMENSION_COUNT = "{arch}.rope.dimension_count"
+ DIMENSION_COUNT_SWA = "{arch}.rope.dimension_count_swa"
+ DIMENSION_SECTIONS = "{arch}.rope.dimension_sections"
+ SECTION_ORDER = "{arch}.rope.section_order"
+ FREQ_BASE = "{arch}.rope.freq_base"
+ FREQ_BASE_SWA = "{arch}.rope.freq_base_swa"
+ SCALING_TYPE = "{arch}.rope.scaling.type"
+ SCALING_FACTOR = "{arch}.rope.scaling.factor"
+ SCALING_ALPHA = "{arch}.rope.scaling.alpha"
+ SCALING_ATTN_FACTOR = "{arch}.rope.scaling.attn_factor"
+ SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length"
+ SCALING_FINETUNED = "{arch}.rope.scaling.finetuned"
+ SCALING_YARN_TRUNCATE = "{arch}.rope.scaling.yarn_truncate"
+ SCALING_YARN_LOG_MUL = "{arch}.rope.scaling.yarn_log_multiplier"
+ SCALING_YARN_EXT_FACTOR = "{arch}.rope.scaling.yarn_ext_factor"
+ SCALING_YARN_ATTN_FACTOR = "{arch}.rope.scaling.yarn_attn_factor"
+ SCALING_YARN_BETA_FAST = "{arch}.rope.scaling.yarn_beta_fast"
+ SCALING_YARN_BETA_SLOW = "{arch}.rope.scaling.yarn_beta_slow"
+
+ class Activation:
+ SITU_BETA = "{arch}.activation.situ_beta"
+ SITU_LINEAR_BETA = "{arch}.activation.situ_linear_beta"
+
+ class AttnRes:
+ BLOCK_SIZE = "{arch}.attn_res.block_size"
+
+ class Split:
+ LLM_KV_SPLIT_NO = "split.no"
+ LLM_KV_SPLIT_COUNT = "split.count"
+ LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"
+
+ class HRM:
+ LAYERS_PER_STACK = "{arch}.hrm.layers_per_stack"
+ H_CYCLES = "{arch}.hrm.h_cycles"
+ L_CYCLES = "{arch}.hrm.l_cycles"
+ PREFIX_LM = "{arch}.hrm.prefix_lm"
+
+ class SSM:
+ CONV_KERNEL = "{arch}.ssm.conv_kernel"
+ INNER_SIZE = "{arch}.ssm.inner_size"
+ STATE_SIZE = "{arch}.ssm.state_size"
+ TIME_STEP_RANK = "{arch}.ssm.time_step_rank"
+ GROUP_COUNT = "{arch}.ssm.group_count"
+ DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms"
+
+ class KDA:
+ HEAD_DIM = "{arch}.kda.head_dim"
+ SAFE_GATE = "{arch}.kda.safe_gate"
+ GATE_LOWER_BOUND = "{arch}.kda.gate_lower_bound"
+
+ class WKV:
+ HEAD_SIZE = "{arch}.wkv.head_size"
+
+ class PosNet:
+ EMBEDDING_LENGTH = "{arch}.posnet.embedding_length"
+ BLOCK_COUNT = "{arch}.posnet.block_count"
+
+ class ConvNext:
+ EMBEDDING_LENGTH = "{arch}.convnext.embedding_length"
+ BLOCK_COUNT = "{arch}.convnext.block_count"
+
+ class Classifier:
+ OUTPUT_LABELS = "{arch}.classifier.output_labels"
+ POOLING_TYPE = "{arch}.classifier.pooling_type"
+ ACTIVATION = "{arch}.classifier.activation"
+
+ class ShortConv:
+ L_CACHE = "{arch}.shortconv.l_cache"
+
+ class Decision:
+ TYPE = "{arch}.decision.type"
+ # note: single-use-case keys can be hard-coded in cpp code
+ BLOCK_COUNT = "{arch}.decision.block_count"
+ ROUTING_BLOCK_COUNT = "{arch}.decision.routing_block_count"
+ HEAD_COUNT = "{arch}.decision.head_count"
+ MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens"
+ TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "<type>" or "<type>.<n_opt bucket>"
+ SCORE_BIAS = "{arch}.decision.score_bias.{n}.{i}" # added to the logit of level i of a score question with n levels
+
+ class Tokenizer:
+ MODEL = "tokenizer.ggml.model"
+ PRE = "tokenizer.ggml.pre"
+ LIST = "tokenizer.ggml.tokens"
+ TOKEN_TYPE = "tokenizer.ggml.token_type"
+ TOKEN_TYPE_COUNT = "tokenizer.ggml.token_type_count" # for BERT-style token types
+ SCORES = "tokenizer.ggml.scores"
+ MERGES = "tokenizer.ggml.merges"
+ BOS_ID = "tokenizer.ggml.bos_token_id"
+ EOS_ID = "tokenizer.ggml.eos_token_id"
+ EOT_ID = "tokenizer.ggml.eot_token_id"
+ EOM_ID = "tokenizer.ggml.eom_token_id"
+ UNK_ID = "tokenizer.ggml.unknown_token_id"
+ SEP_ID = "tokenizer.ggml.seperator_token_id"
+ PAD_ID = "tokenizer.ggml.padding_token_id"
+ MASK_ID = "tokenizer.ggml.mask_token_id"
+ ADD_BOS = "tokenizer.ggml.add_bos_token"
+ ADD_EOS = "tokenizer.ggml.add_eos_token"
+ ADD_SEP = "tokenizer.ggml.add_sep_token"
+ ADD_PREFIX = "tokenizer.ggml.add_space_prefix"
+ REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces"
+ PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap"
+ SUPPRESS_TOKENS = "tokenizer.ggml.suppress_tokens"
+ HF_JSON = "tokenizer.huggingface.json"
+ RWKV = "tokenizer.rwkv.world"
+ CHAT_TEMPLATE = "tokenizer.chat_template"
+ CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"
+ CHAT_TEMPLATES = "tokenizer.chat_templates"
+ # Normalizer constants
+ NORMALIZER_NFC = "tokenizer.ggml.normalizer.nfc"
+ NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
+ NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"
+ # FIM/Infill special tokens constants
+ FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"
+ FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"
+ FIM_MID_ID = "tokenizer.ggml.fim_mid_token_id"
+ FIM_PAD_ID = "tokenizer.ggml.fim_pad_token_id"
+ FIM_REP_ID = "tokenizer.ggml.fim_rep_token_id"
+ FIM_SEP_ID = "tokenizer.ggml.fim_sep_token_id"
+ # deprecated:
+ PREFIX_ID = "tokenizer.ggml.prefix_token_id"
+ SUFFIX_ID = "tokenizer.ggml.suffix_token_id"
+ MIDDLE_ID = "tokenizer.ggml.middle_token_id"
+
+ class Adapter:
+ TYPE = "adapter.type"
+ LORA_ALPHA = "adapter.lora.alpha"
+ LORA_TASK_NAME = "adapter.lora.task_name"
+ LORA_PROMPT_PREFIX = "adapter.lora.prompt_prefix"
+ ALORA_INVOCATION_TOKENS = "adapter.alora.invocation_tokens"
+
+ class IMatrix:
+ CHUNK_COUNT = "imatrix.chunk_count"
+ CHUNK_SIZE = "imatrix.chunk_size"
+ DATASETS = "imatrix.datasets"
+
+ class Clip:
+ PROJECTOR_TYPE = "clip.projector_type"
+ HAS_VISION_ENCODER = "clip.has_vision_encoder"
+ HAS_AUDIO_ENCODER = "clip.has_audio_encoder"
+ HAS_GEN_AUDIO_ENCODER = "clip.has_gen_audio_encoder"
+ HAS_LLAVA_PROJECTOR = "clip.has_llava_projector"
+
+ class ClipVision:
+ PROJECTOR_TYPE = "clip.vision.projector_type" # for mixed modality models
+ IMAGE_SIZE = "clip.vision.image_size"
+ IMAGE_MIN_PIXELS = "clip.vision.image_min_pixels"
+ IMAGE_MAX_PIXELS = "clip.vision.image_max_pixels"
+ PREPROC_MIN_TILES = "clip.vision.preproc_min_tiles"
+ PREPROC_MAX_TILES = "clip.vision.preproc_max_tiles"
+ PREPROC_IMAGE_SIZE = "clip.vision.preproc_image_size"
+ PATCH_SIZE = "clip.vision.patch_size"
+ EMBEDDING_LENGTH = "clip.vision.embedding_length"
+ FEED_FORWARD_LENGTH = "clip.vision.feed_forward_length"
+ PROJECTION_DIM = "clip.vision.projection_dim"
+ BLOCK_COUNT = "clip.vision.block_count"
+ IMAGE_MEAN = "clip.vision.image_mean"
+ IMAGE_STD = "clip.vision.image_std"
+ MAX_SLICE_NUMS = "clip.vision.max_slice_nums"
+ IMAGE_RESIZE_ALGO = "clip.vision.image_resize_algo"
+ SPATIAL_MERGE_SIZE = "clip.vision.spatial_merge_size"
+ SWIGLU_CLAMP = "clip.vision.swiglu_clamp"
+ EXPERT_COUNT_PER_LAYER = "clip.vision.expert_count_per_layer" # dots3note pyramid MoE, 0 = dense layer
+ EXPERT_USED_COUNT = "clip.vision.expert_used_count"
+ USE_GELU = "clip.use_gelu"
+ USE_SILU = "clip.use_silu"
+ N_WA_PATTERN = "clip.vision.n_wa_pattern" # used by qwen2.5vl
+ WA_LAYER_INDEXES = "clip.vision.wa_layer_indexes" # used by youtuvl
+ WA_PATTERN_MODE = "clip.vision.wa_pattern_mode" # used by mimovl, per-layer -1/0/1
+ IS_DEEPSTACK_LAYERS = "clip.vision.is_deepstack_layers"
+ WINDOW_SIZE = "clip.vision.window_size"
+ FEATURE_LAYERS = "clip.vision.feature_layer" # Granite4 Vision
+ IMAGE_GRID_PINPOINTS = "clip.vision.image_grid_pinpoints" # Granite4 Vision
+
+ class Attention:
+ HEAD_COUNT = "clip.vision.attention.head_count"
+ HEAD_COUNT_KV = "clip.vision.attention.head_count_kv" # used by mimovl (GQA)
+ HEAD_DIM = "clip.vision.attention.head_dim" # set when qkv width != n_embd
+ LAYERNORM_EPS = "clip.vision.attention.layer_norm_epsilon"
+
+ class Projector:
+ SCALE_FACTOR = "clip.vision.projector.scale_factor"
+ QUERY_SIDE = "clip.vision.projector.query_side"
+ WINDOW_SIDE = "clip.vision.projector.window_side"
+ SPATIAL_OFFSETS = "clip.vision.projector.spatial_offsets"
+
+ class SAM:
+ BLOCK_COUNT = "clip.vision.sam.block_count"
+ EMBEDDING_LENGTH = "clip.vision.sam.embedding_length"
+ HEAD_COUNT = "clip.vision.sam.head_count"
+
+ class ClipAudio:
+ PROJECTOR_TYPE = "clip.audio.projector_type" # for mixed modality models
+ NUM_MEL_BINS = "clip.audio.num_mel_bins"
+ EMBEDDING_LENGTH = "clip.audio.embedding_length"
+ FEED_FORWARD_LENGTH = "clip.audio.feed_forward_length"
+ PROJECTION_DIM = "clip.audio.projection_dim"
+ BLOCK_COUNT = "clip.audio.block_count"
+ SUBSAMPLING_FACTOR = "clip.audio.subsampling_factor"
+ CHUNK_SIZE = "clip.audio.chunk_size"
+ CONV_KERNEL_SIZE = "clip.audio.conv_kernel_size"
+ MAX_POS_EMB = "clip.audio.max_pos_emb"
+ FEATURE_LAYERS = "clip.audio.feature_layer" # Granite Speech Plus
+ RVQ_NUM_QUANTIZERS = "clip.audio.rvq.num_quantizers"
+ RVQ_CODEBOOK_SIZE = "clip.audio.rvq.codebook_size"
+ WA_PATTERN_MODE = "clip.audio.wa_pattern_mode" # per-layer -1 (full) / 0 (windowed)
+ WINDOW_SIZE = "clip.audio.window_size"
+ LOCAL_BLOCK_COUNT = "clip.audio.local_block_count" # mimo-v2.5: input_local_transformer layer count
+ LOCAL_GROUP_SIZE = "clip.audio.local_group_size" # mimo-v2.5: input_local_transformer grouping size
+
+ class Attention:
+ HEAD_COUNT = "clip.audio.attention.head_count"
+ LAYERNORM_EPS = "clip.audio.attention.layer_norm_epsilon"
+
+ class Projector:
+ STACK_FACTOR = "clip.audio.projector.stack_factor"
+ WINDOW_SIZE = "clip.audio.projector.window_size"
+ DOWNSAMPLE_RATE = "clip.audio.projector.downsample_rate"
+ HEAD_COUNT = "clip.audio.projector.head_count"
+
+ class ClipGenAudio:
+ PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models
+ # name of the weight variant, for settings that are not in the checkpoint
+ MODEL_VARIANT = "clip.gen.audio.model_variant"
+ EMBEDDING_LENGTH = "clip.gen.audio.embedding_length"
+ FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length"
+ BLOCK_COUNT = "clip.gen.audio.block_count"
+ PROJECTION_DIM = "clip.gen.audio.projection_dim"
+
+ class Attention:
+ HEAD_COUNT = "clip.gen.audio.attention.head_count"
+ HEAD_COUNT_KV = "clip.gen.audio.attention.head_count_kv"
+ LAYERNORM_EPS = "clip.gen.audio.attention.layer_norm_epsilon"
+
+ class Diffusion:
+ SHIFT_LOGITS = "diffusion.shift_logits"
+
+ class xIELU:
+ ALPHA_P = "xielu.alpha_p"
+ ALPHA_N = "xielu.alpha_n"
+ BETA = "xielu.beta"
+ EPS = "xielu.eps"
+
+ class PrismHadamard:
+ VERSION = "prism.hadamard.version"
+ TIED_OUTPUT = "prism.hadamard.tied_output"
+ BLOCK_SIZE = "prism.hadamard.block_size"
+ TRANSFORM = "prism.hadamard.transform"
+ AXIS = "prism.hadamard.axis"
+ SIGN_MODE = "prism.hadamard.sign_mode"
+ SIGN_WIDTHS = "prism.hadamard.sign_widths"
+ SIGN_VALUES = "prism.hadamard.sign_values"
+ WEIGHT_NAMES = "prism.hadamard.weight_names"
+ INVERSE_WEIGHT_NAMES = "prism.hadamard.inverse_weight_names"
+ GDN_V_GROUPED = "prism.hadamard.gdn_v_grouped"
+
+
+#
+# recommended mapping of model tensor names for storage in gguf
+#
+
+
+class GGUFType:
+ MODEL = "model"
+ ADAPTER = "adapter"
+ IMATRIX = "imatrix"
+ MMPROJ = "mmproj" # dummy, unused for now
+
+
+class MODEL_ARCH(IntEnum):
+ MMPROJ = auto() # dummy arch for clip.cpp
+ LLAMA = auto()
+ LLAMA4 = auto()
+ DECI = auto()
+ FALCON = auto()
+ FALCON_H1 = auto()
+ BAICHUAN = auto()
+ GROK = auto()
+ GPT2 = auto()
+ GPTJ = auto()
+ GPTNEOX = auto()
+ MPT = auto()
+ STARCODER = auto()
+ REFACT = auto()
+ BERT = auto()
+ MODERN_BERT = auto()
+ NOMIC_BERT = auto()
+ NOMIC_BERT_MOE = auto()
+ NEO_BERT = auto()
+ JINA_BERT_V2 = auto()
+ JINA_BERT_V3 = auto()
+ EUROBERT = auto()
+ BLOOM = auto()
+ STABLELM = auto()
+ QWEN = auto()
+ QWEN2 = auto()
+ QWEN2MOE = auto()
+ QWEN2VL = auto()
+ QWEN3 = auto()
+ QWEN3MOE = auto()
+ QWEN3NEXT = auto()
+ HRM_TEXT = auto()
+ QWEN3VL = auto()
+ QWEN3VLMOE = auto()
+ QWEN35 = auto()
+ QWEN35MOE = auto()
+ CLEF = auto()
+ VLLM_DECISION2 = auto()
+ QWEN4EXP = auto()
+ PHI2 = auto()
+ PHI3 = auto()
+ PHIMOE = auto()
+ PLAMO = auto()
+ PLAMO2 = auto()
+ PLAMO3 = auto()
+ CODESHELL = auto()
+ ORION = auto()
+ INTERNLM2 = auto()
+ MINICPM = auto()
+ MINICPM3 = auto()
+ GEMMA = auto()
+ GEMMA2 = auto()
+ GEMMA3 = auto()
+ GEMMA3N = auto()
+ GEMMA4 = auto()
+ GEMMA4_ASSISTANT = auto()
+ GEMMA_EMBEDDING = auto()
+ GEMMA_EMBEDDING2 = auto()
+ STARCODER2 = auto()
+ RWKV6 = auto()
+ RWKV6QWEN2 = auto()
+ RWKV7 = auto()
+ ARWKV7 = auto()
+ MAMBA = auto()
+ MAMBA2 = auto()
+ MAPLE = auto()
+ JAMBA = auto()
+ XVERSE = auto()
+ COMMAND_R = auto()
+ COHERE2 = auto()
+ COHERE2MOE = auto()
+ DBRX = auto()
+ OLMO = auto()
+ OLMO2 = auto()
+ OLMOE = auto()
+ MUSE_GLIMMER = auto()
+ OPENELM = auto()
+ ARCTIC = auto()
+ DEEPSEEK = auto()
+ DEEPSEEK2 = auto()
+ DEEPSEEK2OCR = auto()
+ DEEPSEEK32 = auto()
+ DEEPSEEK4 = auto()
+ CHATGLM = auto()
+ GLM4 = auto()
+ GLM4_MOE = auto()
+ GLM_DSA = auto()
+ GLM5_NEXT = auto()
+ BITNET = auto()
+ T5 = auto()
+ T5ENCODER = auto()
+ JAIS = auto()
+ JAIS2 = auto()
+ NEMOTRON = auto()
+ NEMOTRON_H = auto()
+ NEMOTRON_H_MOE = auto()
+ EXAONE = auto()
+ EXAONE4 = auto()
+ EXAONE_MOE = auto()
+ GRANITE = auto()
+ GRANITE_MOE = auto()
+ GRANITE_HYBRID = auto()
+ GRANITE_SWITCH = auto()
+ GRANITE_SWA = auto()
+ CHAMELEON = auto()
+ WAVTOKENIZER_DEC = auto()
+ PLM = auto()
+ BAILINGMOE = auto()
+ BAILINGMOE2 = auto()
+ BAILINGMOE3 = auto()
+ DOTS1 = auto()
+ DOTS3NOTE = auto()
+ ARCEE = auto()
+ AFMOE = auto()
+ LAGUNA = auto()
+ ERNIE4_5 = auto()
+ ERNIE4_5_MOE = auto()
+ HUNYUAN_MOE = auto()
+ HUNYUAN_DENSE = auto()
+ HUNYUAN_VL = auto()
+ HY_V3 = auto()
+ HY_V4 = auto()
+ SMOLLM3 = auto()
+ GPT_OSS = auto()
+ LFM2 = auto()
+ LFM2MOE = auto()
+ DREAM = auto()
+ SMALLTHINKER = auto()
+ LLADA = auto()
+ LLADA_MOE = auto()
+ SEED_OSS = auto()
+ GROVEMOE = auto()
+ APERTUS = auto()
+ COGVLM = auto()
+ MINIMAX01 = auto()
+ MINIMAXM2 = auto()
+ MINIMAXM3 = auto()
+ RND1 = auto()
+ PANGU_EMBED = auto()
+ MISTRAL3 = auto()
+ EAGLE3 = auto()
+ DFLASH = auto()
+ MISTRAL4 = auto()
+ PADDLEOCR = auto()
+ MIMO2 = auto()
+ STEP35 = auto()
+ SPARK2_5 = auto()
+ LLAMA_EMBED = auto()
+ MAINCODER = auto()
+ KIMI_LINEAR = auto()
+ KIMI_K3 = auto()
+ TALKIE = auto()
+ MELLUM = auto()
+ NANBEIGE = auto()
+ QWEN3TTS = auto()
+ POCKETTTS = auto()
+ K2HORIZON = auto()
+
+
+class VISION_PROJECTOR_TYPE(IntEnum):
+ MLP = auto()
+ LDP = auto()
+ LDPV2 = auto()
+ RESAMPLER = auto()
+ GLM_EDGE = auto()
+ MERGER = auto()
+ GEMMA3N = auto()
+ GEMMA3 = auto()
+ QWEN3VL = auto()
+ LING3VL = auto()
+ STEP3VL = auto()
+ COGVLM = auto()
+
+
+class MODEL_TENSOR(IntEnum):
+ TOKEN_EMBD = auto()
+ TOKEN_EMBD_NORM = auto()
+ MASKED_EMBD_CENTROIDS= auto()
+ MASKED_EMBD_ORDERING = auto()
+ TOKEN_TYPES = auto()
+ POS_EMBD = auto()
+ OUTPUT = auto()
+ HRM_Z_L_INIT = auto()
+ DENSE_2_OUT = auto() # embeddinggemma 2_Dense
+ DENSE_3_OUT = auto() # embeddinggemma 3_Dense
+ OUTPUT_NORM = auto()
+ HC_HEAD_FN = auto()
+ HC_HEAD_BASE = auto()
+ HC_HEAD_SCALE = auto()
+ HC_HEAD_NORM = auto() # qwen4exp
+ HC_HEAD_DOWN = auto() # qwen4exp
+ HC_HEAD_UP = auto() # qwen4exp
+ ROPE_FREQS = auto()
+ ROPE_FACTORS_LONG = auto()
+ ROPE_FACTORS_SHORT = auto()
+ ATTN_Q = auto()
+ ATTN_K = auto()
+ ATTN_V = auto()
+ ATTN_QKV = auto()
+ ATTN_OUT = auto()
+ ATTN_NORM = auto()
+ ATTN_NORM_2 = auto()
+ ATTN_OUT_NORM = auto()
+ ATTN_POST_NORM = auto()
+ ATTN_ROT_EMBD = auto()
+ ATTN_SINKS = auto()
+ ATTN_GATE = auto()
+ FFN_GATE_INP = auto()
+ FFN_GATE_INP_SHEXP = auto()
+ FFN_NORM = auto()
+ FFN_PRE_NORM = auto() # alias of FFN_NORM
+ FFN_PRE_NORM_2 = auto() # gemma4
+ FFN_POST_NORM = auto()
+ FFN_POST_NORM_1 = auto() # gemma4
+ FFN_POST_NORM_2 = auto() # gemma4
+ FFN_GATE = auto()
+ FFN_DOWN = auto()
+ FFN_UP = auto()
+ FFN_ACT = auto()
+ FFN_NORM_EXP = auto()
+ FFN_GATE_EXP = auto()
+ FFN_DOWN_EXP = auto()
+ FFN_UP_EXP = auto()
+ FFN_GATE_UP_EXP = auto()
+ FFN_GATE_SHEXP = auto()
+ FFN_DOWN_SHEXP = auto()
+ FFN_UP_SHEXP = auto()
+ FFN_GATE_CHEXP = auto()
+ FFN_DOWN_CHEXP = auto()
+ FFN_UP_CHEXP = auto()
+ FFN_EXP_PROBS_B = auto()
+ FFN_EXP_PROBS_B_VL = auto() # deepseek4 vision (bias for image tokens)
+ FFN_GATE_TID2EID = auto()
+ MOE_LATENT_DOWN = auto() # nemotron 3 super
+ MOE_LATENT_UP = auto() # nemotron 3 super
+ ATTN_Q_NORM = auto()
+ ATTN_K_NORM = auto()
+ LAYER_OUT_NORM = auto()
+ LAYER_OUT_SCALE = auto()
+ PER_LAYER_TOKEN_EMBD = auto() # gemma3n
+ PER_LAYER_MODEL_PROJ = auto() # gemma3n
+ PER_LAYER_INP_GATE = auto() # gemma3n
+ PER_LAYER_PROJ = auto() # gemma3n
+ PER_LAYER_PROJ_NORM = auto() # gemma3n
+ PER_LAYER_POST_NORM = auto() # gemma3n
+ ALTUP_PROJ = auto() # gemma3n
+ ALTUP_UNEMBD_PROJ = auto() # gemma3n
+ ALTUP_CORRECT_COEF = auto() # gemma3n
+ ALTUP_CORRECT_SCALE = auto() # gemma3n
+ ALTUP_PREDICT_COEF = auto() # gemma3n
+ ALTUP_ROUTER = auto() # gemma3n
+ ALTUP_ROUTER_NORM = auto() # gemma3n
+ LAUREL_L = auto() # gemma3n
+ LAUREL_R = auto() # gemma3n
+ LAUREL_POST_NORM = auto() # gemma3n
+ SSM_IN = auto()
+ SSM_CONV1D = auto()
+ SSM_X = auto()
+ SSM_DT = auto()
+ SSM_DT_NORM = auto()
+ SSM_A = auto()
+ SSM_B_NORM = auto()
+ SSM_C_NORM = auto()
+ SSM_D = auto()
+ SSM_NORM = auto()
+ SSM_OUT = auto()
+ SSM_ALPHA = auto() # qwen3.5
+ SSM_BETA_ALPHA = auto() # qwen3next
+ SSM_CONV1D_Q = auto() # Kimi Linear
+ SSM_CONV1D_K = auto() # Kimi Linear
+ SSM_CONV1D_V = auto() # Kimi Linear
+ SSM_F_A = auto() # Kimi Linear
+ SSM_F_B = auto() # Kimi Linear
+ SSM_BETA = auto() # Kimi Linear qwen3.5
+ SSM_G_A = auto() # Kimi Linear
+ SSM_G_B = auto() # Kimi Linear
+ SSM_G = auto() # Kimi K3 (full-rank KDA gate, replaces SSM_G_A/SSM_G_B)
+ ATTN_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, pre-attention)
+ FFN_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, pre-FFN)
+ OUTPUT_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, final)
+ FFN_ROUTED_DOWN = auto() # Kimi K3 (latent MoE: hidden -> latent)
+ FFN_ROUTED_UP = auto() # Kimi K3 (latent MoE: latent -> hidden)
+ FFN_ROUTED_NORM = auto() # Kimi K3 (latent MoE: norm on expert output)
+ TIME_MIX_W0 = auto()
+ TIME_MIX_W1 = auto()
+ TIME_MIX_W2 = auto()
+ TIME_MIX_A0 = auto()
+ TIME_MIX_A1 = auto()
+ TIME_MIX_A2 = auto()
+ TIME_MIX_V0 = auto()
+ TIME_MIX_V1 = auto()
+ TIME_MIX_V2 = auto()
+ TIME_MIX_G1 = auto()
+ TIME_MIX_G2 = auto()
+ TIME_MIX_K_K = auto()
+ TIME_MIX_K_A = auto()
+ TIME_MIX_R_K = auto()
+ TIME_MIX_LERP_X = auto()
+ TIME_MIX_LERP_K = auto()
+ TIME_MIX_LERP_V = auto()
+ TIME_MIX_LERP_R = auto()
+ TIME_MIX_LERP_G = auto()
+ TIME_MIX_LERP_FUSED = auto()
+ TIME_MIX_LERP_W = auto()
+ TIME_MIX_FIRST = auto()
+ TIME_MIX_DECAY = auto()
+ TIME_MIX_DECAY_W1 = auto()
+ TIME_MIX_DECAY_W2 = auto()
+ TIME_MIX_KEY = auto()
+ TIME_MIX_VALUE = auto()
+ TIME_MIX_RECEPTANCE = auto()
+ TIME_MIX_GATE = auto()
+ TIME_MIX_LN = auto()
+ TIME_MIX_OUTPUT = auto()
+ CHANNEL_MIX_LERP_K = auto()
+ CHANNEL_MIX_LERP_R = auto()
+ CHANNEL_MIX_KEY = auto()
+ CHANNEL_MIX_RECEPTANCE = auto()
+ CHANNEL_MIX_VALUE = auto()
+ ATTN_Q_A = auto()
+ ATTN_Q_B = auto()
+ ATTN_KV_A_MQA = auto()
+ ATTN_KV_B = auto()
+ ATTN_K_B = auto()
+ ATTN_V_B = auto()
+ ATTN_Q_A_NORM = auto()
+ ATTN_KV_A_NORM = auto()
+ ATTN_KV = auto()
+ ATTN_KV_NORM = auto()
+ ATTN_OUT_A = auto()
+ ATTN_OUT_B = auto()
+ HC_ATTN_FN = auto()
+ HC_ATTN_BASE = auto()
+ HC_ATTN_SCALE = auto()
+ HC_FFN_FN = auto()
+ HC_FFN_BASE = auto()
+ HC_FFN_SCALE = auto()
+ HC_ATTN_NORM = auto() # qwen4exp
+ HC_ATTN_DOWN = auto() # qwen4exp
+ HC_ATTN_UP = auto() # qwen4exp
+ HC_ATTN_INJECT = auto() # qwen4exp
+ HC_FFN_NORM = auto() # qwen4exp
+ HC_FFN_DOWN = auto() # qwen4exp
+ HC_FFN_UP = auto() # qwen4exp
+ HC_FFN_INJECT = auto() # qwen4exp
+ PLE_KEY = auto() # qwen4exp
+ PLE_VALUE = auto() # qwen4exp
+ PLE_NORM_KEY = auto() # qwen4exp
+ PLE_NORM_QUERY = auto() # qwen4exp
+ PLE_NORM_CONV = auto() # qwen4exp
+ PLE_CONV1D = auto() # qwen4exp
+ ATTN_COMPRESSOR_WKV = auto()
+ ATTN_COMPRESSOR_WGATE = auto()
+ ATTN_COMPRESSOR_APE = auto()
+ ATTN_COMPRESSOR_NORM = auto()
+ FFN_SUB_NORM = auto()
+ ATTN_SUB_NORM = auto()
+ DEC_ATTN_NORM = auto()
+ DEC_ATTN_Q = auto()
+ DEC_ATTN_K = auto()
+ DEC_ATTN_V = auto()
+ DEC_ATTN_OUT = auto()
+ DEC_ATTN_REL_B = auto()
+ DEC_CROSS_ATTN_NORM = auto()
+ DEC_CROSS_ATTN_NORM_KV = auto()
+ DEC_CROSS_ATTN_Q = auto()
+ DEC_CROSS_ATTN_K = auto()
+ DEC_CROSS_ATTN_V = auto()
+ DEC_CROSS_ATTN_OUT = auto()
+ DEC_CROSS_ATTN_REL_B = auto()
+ DEC_FFN_NORM = auto()
+ DEC_FFN_GATE = auto()
+ DEC_FFN_DOWN = auto()
+ DEC_FFN_UP = auto()
+ DEC_OUTPUT_NORM = auto()
+ ENC_ATTN_NORM = auto()
+ ENC_ATTN_Q = auto()
+ ENC_ATTN_K = auto()
+ ENC_ATTN_V = auto()
+ ENC_ATTN_OUT = auto()
+ ENC_ATTN_REL_B = auto()
+ ENC_FFN_NORM = auto()
+ ENC_FFN_GATE = auto()
+ ENC_FFN_DOWN = auto()
+ ENC_FFN_UP = auto()
+ ENC_OUTPUT_NORM = auto()
+ CLS = auto() # classifier
+ CLS_OUT = auto() # classifier output projection
+ CLS_NORM = auto()
+ DECISION_HIDDEN_NORM = auto()
+ DECISION_PROJ_MEMORY = auto()
+ DECISION_PROJ_QUESTION = auto()
+ DECISION_PROJ_OPTION_QUESTION = auto()
+ DECISION_PROJ_GLOBAL = auto()
+ DECISION_PROJ_OPTION_CONTEXT = auto()
+ DECISION_PROJ_OPTION_LEXICAL = auto()
+ DECISION_OPTION_SUMMARY_NORM = auto()
+ DECISION_FIELD_NORM = auto()
+ DECISION_OPTION_NORM = auto()
+ DECISION_SCALES = auto()
+ DECISION_SCORER = auto()
+ DECISION_SCORER_OUT = auto()
+ DECISION_QUERY_NORM = auto()
+ DECISION_KEY = auto()
+ DECISION_QUERY = auto()
+ DECISION_OPTION_MLP = auto()
+ DECISION_QUERY_MLP = auto()
+ CONV1D = auto()
+ CONVNEXT_DW = auto()
+ CONVNEXT_NORM = auto()
+ CONVNEXT_PW1 = auto()
+ CONVNEXT_PW2 = auto()
+ CONVNEXT_GAMMA = auto()
+ POSNET_CONV1 = auto()
+ POSNET_CONV2 = auto()
+ POSNET_NORM = auto()
+ POSNET_NORM1 = auto()
+ POSNET_NORM2 = auto()
+ POSNET_ATTN_NORM = auto()
+ POSNET_ATTN_Q = auto()
+ POSNET_ATTN_K = auto()
+ POSNET_ATTN_V = auto()
+ POSNET_ATTN_OUT = auto()
+ SHORTCONV_CONV = auto()
+ SHORTCONV_INPROJ = auto()
+ SHORTCONV_OUTPROJ = auto()
+ VISEXP_ATTN_QKV = auto()
+ VISEXP_ATTN_OUT = auto()
+ VISEXP_GATE = auto()
+ VISEXP_DOWN = auto()
+ VISEXP_UP = auto()
+ INDEXER_K_NORM = auto()
+ INDEXER_PROJ = auto()
+ INDEXER_ATTN_K = auto()
+ INDEXER_ATTN_Q_B = auto()
+ INDEXER_Q_PROJ = auto()
+ INDEXER_K_PROJ = auto()
+ INDEXER_Q_NORM = auto()
+ INDEXER_COMPRESSOR_WKV = auto()
+ INDEXER_COMPRESSOR_WGATE = auto()
+ INDEXER_COMPRESSOR_APE = auto()
+ INDEXER_COMPRESSOR_NORM = auto()
+ INDEXER_KPOOL_GATE = auto()
+ INDEXER_KPOOL_APE = auto()
+ ATTN_V_GATE = auto() # k2-horizon MoVA router
+ ATTN_V_EXP = auto() # k2-horizon MoVA value experts
+ # vision
+ V_MMPROJ = auto()
+ V_MMPROJ_FC = auto()
+ V_MMPROJ_MLP = auto()
+ V_MMPROJ_PEG = auto()
+ V_ENC_EMBD_CLS = auto()
+ V_ENC_EMBD_PATCH = auto()
+ V_ENC_EMBD_NORM = auto()
+ V_ENC_EMBD_PATCH_NORM = auto() # allow multiple norms in the same embd, e.g. for gemma4u
+ V_ENC_EMBD_POS = auto()
+ V_ENC_INPUT_NORM = auto()
+ V_ENC_ATTN_QKV = auto()
+ V_ENC_ATTN_Q = auto()
+ V_ENC_ATTN_Q_NORM = auto()
+ V_ENC_ATTN_K = auto()
+ V_ENC_ATTN_K_NORM = auto()
+ V_ENC_ATTN_V = auto()
+ V_ENC_ATTN_O = auto()
+ V_ENC_ATTN_O_NORM = auto()
+ V_ENC_ATTN_SINKS = auto() # mimovl
+ V_ENC_POST_ATTN_NORM = auto()
+ V_ENC_FFN_UP = auto()
+ V_ENC_FFN_GATE = auto()
+ V_ENC_FFN_DOWN = auto()
+ V_ENC_FFN_GATE_INP = auto() # dots3note vision MoE router
+ V_ENC_FFN_GATE_EXPS = auto()
+ V_ENC_FFN_UP_EXPS = auto()
+ V_ENC_FFN_DOWN_EXPS = auto()
+ V_ENC_FFN_EXP_PROBS_B = auto()
+ V_ENC_ATTN_POST_NORM = auto() # gemma4
+ V_ENC_FFN_POST_NORM = auto()
+ V_LAYER_SCALE_1 = auto()
+ V_LAYER_SCALE_2 = auto()
+ V_LAYER_OUT_SCALE = auto()
+ V_PRE_NORM = auto()
+ V_POST_NORM = auto()
+ V_MM_PRE_NORM = auto() # hunyuanvl
+ V_MM_POST_NORM = auto()
+ V_MM_INP_NORM = auto()
+ V_MM_INP_PROJ = auto() # gemma3
+ V_MM_SOFT_EMB_NORM = auto() # gemma3
+ V_MM_EMBEDDING = auto() # gemma3n
+ V_MM_HARD_EMB_NORM = auto() # gemma3n
+ V_ENC_CONV_STEM = auto() # gemma3n
+ V_ENC_CONV_STEM_NORM = auto() # gemma3n
+ V_ENC_MSFA_EXP = auto() # gemma3n
+ V_ENC_MSFA_EXP_NORM = auto() # gemma3n
+ V_ENC_MSFA_PROJ = auto() # gemma3n
+ V_ENC_MSFA_PROJ_NORM = auto() # gemma3n
+ V_ENC_MSFA_NORM = auto() # gemma3n
+ V_RESMPL_POS_EMBD_K = auto() # minicpmv
+ V_RESMPL_ATTN_Q = auto() # minicpmv
+ V_RESMPL_ATTN_K = auto() # minicpmv
+ V_RESMPL_ATTN_V = auto() # minicpmv
+ V_RESMPL_ATTN_OUT = auto() # minicpmv
+ V_RESMPL_KV = auto() # minicpmv
+ V_RESMPL_KV_NORM = auto() # minicpmv
+ V_RESMPL_POST_NORM = auto() # minicpmv
+ V_RESMPL_Q_NORM = auto() # minicpmv
+ V_RESMPL_PROJ = auto() # minicpmv
+ V_RESMPL_QUERY = auto() # minicpmv
+ V_TOK_EMBD_IMG_BREAK = auto() # pixtral
+ V_TOK_EMBD_IMG_START = auto() # deepseek4v
+ V_TOK_EMBD_IMG_END = auto() # deepseek4v
+ V_TOK_EMBD_IMG_PAD = auto() # deepseek4v
+ V_MM_PATCH_MERGER = auto() # mistral small 3.1
+ V_DS_NORM = auto() # qwen3vl
+ V_DS_FC1 = auto() # qwen3vl
+ V_DS_FC2 = auto() # qwen3vl
+ V_MERGER_LN1 = auto() # minicpmv4_6
+ V_MERGER_ATTN_Q = auto() # minicpmv4_6
+ V_MERGER_ATTN_K = auto() # minicpmv4_6
+ V_MERGER_ATTN_V = auto() # minicpmv4_6
+ V_MERGER_ATTN_O = auto() # minicpmv4_6
+ V_MERGER_DS_LN = auto() # minicpmv4_6
+ V_MERGER_DS_UP = auto() # minicpmv4_6
+ V_MERGER_DS_DOWN = auto() # minicpmv4_6
+ V_MM_POST_FC_NORM = auto() # cogvlm
+ V_MM_UP = auto() # cogvlm
+ V_MM_DOWN = auto() # cogvlm
+ V_MM_GATE = auto() # cogvlm
+ V_MM_MERGER_FC1 = auto() # minimax-m3 (patch-merge MLP)
+ V_MM_MERGER_FC2 = auto() # minimax-m3 (patch-merge MLP)
+ V_TOK_BOI = auto() # cogvlm
+ V_TOK_EOI = auto() # cogvlm
+ V_TOK_IMG_BEGIN = auto() # hunyuanvl
+ V_TOK_IMG_END = auto() # hunyuanvl
+ V_STD_BIAS = auto() # gemma4
+ V_STD_SCALE = auto() # gemma4
+ V_SAM_POS_EMBD = auto() # Deepseek-OCR
+ V_SAM_PATCH_EMBD = auto() # Deepseek-OCR
+ V_SAM_PRE_NORM = auto() # Deepseek-OCR
+ V_SAM_POST_NORM = auto() # Deepseek-OCR
+ V_SAM_ATTN_POS_H = auto() # Deepseek-OCR
+ V_SAM_ATTN_POS_W = auto() # Deepseek-OCR
+ V_SAM_ATTN_QKV = auto() # Deepseek-OCR
+ V_SAM_ATTN_OUT = auto() # Deepseek-OCR
+ V_SAM_MLP_LIN_1 = auto() # Deepseek-OCR
+ V_SAM_MLP_LIN_2 = auto() # Deepseek-OCR
+ V_SAM_NECK = auto() # Deepseek-OCR
+ V_SAM_NET_2 = auto() # Deepseek-OCR
+ V_SAM_NET_3 = auto() # Deepseek-OCR
+ V_ENC_EMBD_IMGNL = auto() # Deepseek-OCR
+ V_ENC_EMBD_VSEP = auto() # Deepseek-OCR
+ V_TOK_EMBD_SEP = auto() # MiniCPM-V 4.7
+ V_RESMPL_QUERY_768 = auto() # Deepseek-OCR-2
+ V_RESMPL_QUERY_1024 = auto() # Deepseek-OCR-2
+
+ # qformer projector (vision) - Granite4 Vision
+ V_QF_PROJ_QUERY = auto()
+ V_QF_PROJ_NORM = auto()
+ V_QF_PROJ_LINEAR = auto()
+ V_QF_SELF_ATTN_Q = auto()
+ V_QF_SELF_ATTN_K = auto()
+ V_QF_SELF_ATTN_V = auto()
+ V_QF_SELF_ATTN_O = auto()
+ V_QF_SELF_ATTN_NORM = auto()
+ V_QF_CROSS_ATTN_Q = auto()
+ V_QF_CROSS_ATTN_K = auto()
+ V_QF_CROSS_ATTN_V = auto()
+ V_QF_CROSS_ATTN_O = auto()
+ V_QF_CROSS_ATTN_NORM = auto()
+ V_QF_FFN_UP = auto()
+ V_QF_FFN_DOWN = auto()
+ V_QF_FFN_NORM = auto()
+ V_PROJ_NORM = auto()
+ # multi-projector (bid => projector id) - Granite4 vision
+ V_MULTI_PROJ_IMG_POS = auto()
+ V_MULTI_PROJ_QUERY = auto()
+ V_MULTI_PROJ_NORM = auto()
+ V_MULTI_PROJ_LINEAR = auto()
+ V_MULTI_PROJ_POST_NORM = auto()
+
+ # audio (mtmd)
+ A_ENC_EMBD_POS = auto()
+ A_ENC_EMBD_NORM = auto()
+ A_ENC_EMBD_TO_LOGITS = auto() # lfm2
+ A_ENC_INP_PROJ = auto() # gemma4
+ A_ENC_CONV1D = auto()
+ A_ENC_CONV1D_NORM = auto() # gemma3n
+ A_ENC_CONV2D = auto()
+ A_ENC_CONV_OUT = auto()
+ A_PRE_NORM = auto()
+ A_POST_NORM = auto()
+ A_ENC_LAYER_PRE_NORM = auto() # gemma3n
+ A_ENC_ATTN_Q = auto()
+ A_ENC_ATTN_K = auto()
+ A_ENC_ATTN_V = auto()
+ A_ENC_ATTN_POST_NORM = auto()
+ A_ENC_ATTN_PRE_NORM = auto()
+ A_ENC_ATTN_K_REL = auto() # gemma4
+ A_ENC_PER_DIM_SCALE = auto() # gemma3n
+ A_ENC_INPUT_NORM = auto()
+ A_ENC_OUTPUT = auto() # TODO @ngxson: rename to ATTN_OUT
+ A_ENC_OUTPUT_NORM = auto() # TODO @ngxson: rename to ATTN_OUT
+ A_ENC_FFN_UP = auto()
+ A_ENC_FFN_NORM = auto()
+ A_ENC_FFN_POST_NORM = auto() # gemma3n
+ A_ENC_FFN_SCALE = auto() # gemma3n
+ A_ENC_FFN_GATE = auto()
+ A_ENC_FFN_DOWN = auto()
+ A_ENC_FFN_UP_1 = auto() # lfm2, gemma3n
+ A_ENC_FFN_NORM_1 = auto() # lfm2, gemma3n (pre-norm)
+ A_ENC_FFN_POST_NORM_1 = auto() # gemma3n
+ A_ENC_FFN_SCALE_1 = auto() # gemma3n
+ A_ENC_FFN_GATE_1 = auto() # lfm2, gemma3n
+ A_ENC_FFN_DOWN_1 = auto() # lfm2, gemma3n
+ A_ENC_DOWNSAMPLE_CONV = auto() # mimo-audio-tokenizer: post-transformer downsample conv
+ A_ENC_DOWNSAMPLE_NORM = auto() # mimo-audio-tokenizer: post-transformer downsample norm
+ A_ENC_RVQ_CODEBOOK = auto() # mimo-audio-tokenizer: residual vector quantizer codebook, per quantizer index
+ A_ENC_CONV_RES2 = auto() # qwen3tts
+ A_ENC_SE_CONV1 = auto() # qwen3tts
+ A_ENC_SE_CONV2 = auto() # qwen3tts
+ A_ENC_ASP_ATTN = auto() # qwen3tts
+ A_ENC_ASP_TDNN = auto() # qwen3tts
+ # qwen3tts code_predictor: predicts the remaining RVQ codebooks
+ A_GEN_CODE_PROJ_IN = auto() # small_to_mtp_projection
+ A_GEN_CODE_EMBD = auto() # per-codebook embedding table, merged 3D [n_codebooks, vocab, dim]
+ A_GEN_CODE_HEAD = auto() # per-codebook output head, merged 3D [n_codebooks, vocab, dim]
+ A_GEN_CODE_OUT_EMBD = auto() # codebook-0 embedding, re-fed into the talker backbone (talker.model.codec_embedding)
+ A_GEN_CODE_ATTN_NORM = auto()
+ A_GEN_CODE_ATTN_Q = auto()
+ A_GEN_CODE_ATTN_Q_NORM = auto()
+ A_GEN_CODE_ATTN_K = auto()
+ A_GEN_CODE_ATTN_K_NORM = auto()
+ A_GEN_CODE_ATTN_V = auto()
+ A_GEN_CODE_ATTN_OUT = auto()
+ A_GEN_CODE_FFN_NORM = auto()
+ A_GEN_CODE_FFN_GATE = auto()
+ A_GEN_CODE_FFN_UP = auto()
+ A_GEN_CODE_FFN_DOWN = auto()
+ A_GEN_CODE_OUTPUT_NORM = auto()
+ # qwen3tts code2wav: RVQ codes -> raw PCM
+ A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
+ A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj
+ A_GEN_WAV_QUANT_FIRST_CB = auto() # semantic RVQ codebook (1 layer), folded from embedding_sum/cluster_usage
+ A_GEN_WAV_QUANT_REST_IN = auto() # acoustic RVQ, in_proj
+ A_GEN_WAV_QUANT_REST_OUT = auto() # acoustic RVQ, out_proj
+ A_GEN_WAV_QUANT_REST_CB = auto() # acoustic RVQ codebooks, merged 3D [15, vocab, dim]
+ A_GEN_WAV_PRE_CONV = auto()
+ A_GEN_WAV_TFM_IN_PROJ = auto()
+ A_GEN_WAV_TFM_OUT_PROJ = auto()
+ A_GEN_WAV_TFM_OUTPUT_NORM = auto()
+ A_GEN_WAV_TFM_ATTN_NORM = auto()
+ A_GEN_WAV_TFM_ATTN_Q = auto()
+ A_GEN_WAV_TFM_ATTN_K = auto()
+ A_GEN_WAV_TFM_ATTN_V = auto()
+ A_GEN_WAV_TFM_ATTN_OUT = auto()
+ A_GEN_WAV_TFM_ATTN_SCALE = auto() # layer scale (gamma) on the attn output
+ A_GEN_WAV_TFM_FFN_NORM = auto()
+ A_GEN_WAV_TFM_FFN_GATE = auto()
+ A_GEN_WAV_TFM_FFN_UP = auto()
+ A_GEN_WAV_TFM_FFN_DOWN = auto()
+ A_GEN_WAV_TFM_FFN_SCALE = auto() # layer scale (gamma) on the FFN output
+ A_GEN_WAV_UP_CONV = auto() # causal ConvTranspose1d, 2x upsample
+ A_GEN_WAV_UP_DWCONV = auto() # ConvNeXt depthwise conv
+ A_GEN_WAV_UP_NORM = auto() # ConvNeXt LayerNorm
+ A_GEN_WAV_UP_PW1 = auto() # ConvNeXt pointwise conv 1 (expand)
+ A_GEN_WAV_UP_PW2 = auto() # ConvNeXt pointwise conv 2 (project)
+ A_GEN_WAV_UP_GAMMA = auto() # ConvNeXt layer scale
+ A_GEN_WAV_DAC_ENTRY = auto() # DAC conv_pre
+ A_GEN_WAV_DAC_UP_SNAKE = auto() # DAC per-block SnakeBeta before the upsample conv
+ A_GEN_WAV_DAC_UP_CONV = auto() # DAC per-block causal ConvTranspose1d
+ A_GEN_WAV_DAC_RES_ACT1 = auto() # DAC residual unit, SnakeBeta before conv1
+ A_GEN_WAV_DAC_RES_CONV1 = auto() # DAC residual unit, dilated causal conv
+ A_GEN_WAV_DAC_RES_ACT2 = auto() # DAC residual unit, SnakeBeta before conv2
+ A_GEN_WAV_DAC_RES_CONV2 = auto() # DAC residual unit, pointwise causal conv
+ A_GEN_WAV_DAC_POST_SNAKE = auto() # DAC final SnakeBeta
+ A_GEN_WAV_DAC_POST_CONV = auto() # DAC conv_post -> 1-channel PCM
+ # pocket-tts: SEANet encoder (speaker path) and decoder (a.gen.wav path)
+ A_ENC_SEANET_CONV_IN = auto()
+ A_ENC_SEANET_CONV_OUT = auto()
+ A_ENC_SEANET_RES_CONV1 = auto() # residual unit, dilated conv
+ A_ENC_SEANET_RES_CONV2 = auto() # residual unit, pointwise conv
+ A_ENC_SEANET_SCALE_CONV = auto() # strided downsample conv
+ A_ENC_ATTN_SCALE = auto() # layer scale (gamma) on the attn output
+ A_ENC_FFN_SCALE_LS = auto() # layer scale (gamma) on the FFN output
+ A_ENC_SPEAKER_PROJ = auto() # voice latent -> backbone embd
+ A_GEN_FLOW_INPUT_PROJ = auto()
+ A_GEN_FLOW_COND_EMBD = auto()
+ A_GEN_FLOW_TIME_FREQS = auto() # timestep embedder, stored cos/sin frequencies
+ A_GEN_FLOW_TIME_UP = auto()
+ A_GEN_FLOW_TIME_DOWN = auto()
+ A_GEN_FLOW_TIME_NORM = auto() # RMSNorm alpha
+ A_GEN_FLOW_BLK_NORM = auto() # AdaLN res block, in_ln
+ A_GEN_FLOW_BLK_UP = auto()
+ A_GEN_FLOW_BLK_DOWN = auto()
+ A_GEN_FLOW_BLK_ADA = auto() # AdaLN modulation, -> shift/scale/gate
+ A_GEN_FLOW_FINAL_ADA = auto() # final layer AdaLN modulation, -> shift/scale
+ A_GEN_FLOW_FINAL_PROJ = auto()
+ A_GEN_OUT_EOS = auto() # end-of-speech head on the backbone hidden state
+ A_GEN_INPUT_LINEAR = auto() # generated latent -> backbone embd
+ A_GEN_EMB_MEAN = auto() # latent denormalization stats
+ A_GEN_EMB_STD = auto()
+ A_GEN_WAV_QUANT_OUT = auto() # DummyQuantizer output_proj, latent -> decoder dim
+ A_GEN_WAV_UPSAMPLE = auto() # frame rate -> encoder frame rate, depthwise convtr
+ A_GEN_WAV_SEANET_CONV_IN = auto()
+ A_GEN_WAV_SEANET_CONV_OUT = auto() # -> 1-channel PCM
+ A_GEN_WAV_SEANET_RES_CONV1 = auto()
+ A_GEN_WAV_SEANET_RES_CONV2 = auto()
+ A_GEN_WAV_SEANET_SCALE_CONV = auto() # strided upsample convtr
+ A_MMPROJ = auto()
+ A_MMPROJ_FC = auto()
+ A_MM_NORM_PRE = auto()
+ A_MM_NORM_MID = auto()
+ A_MM_EMBEDDING = auto() # gemma3n
+ A_MM_HARD_EMB_NORM = auto() # gemma3n
+ A_MM_SOFT_EMB_NORM = auto() # gemma3n
+ A_MM_INP_PROJ = auto() # gemma3n
+ A_MM_CODE_EMBD = auto() # mimo: text-side RVQ code embedding table ("text codebook"), merged 3D [n_channels, vocab, dim]
+ A_MM_LOCAL_ATTN_Q = auto() # mimo: input_local_transformer (LLM-side connector)
+ A_MM_LOCAL_ATTN_K = auto()
+ A_MM_LOCAL_ATTN_V = auto()
+ A_MM_LOCAL_ATTN_OUT = auto()
+ A_MM_LOCAL_FFN_GATE = auto()
+ A_MM_LOCAL_FFN_UP = auto()
+ A_MM_LOCAL_FFN_DOWN = auto()
+ A_MM_LOCAL_LN1 = auto()
+ A_MM_LOCAL_LN2 = auto()
+ A_MM_LOCAL_NORM = auto() # final norm after all input_local_transformer layers
+ A_PER_DIM_K_SCALE = auto() # gemma4
+ A_PER_DIM_SCALE = auto() # gemma4
+ # nextn/mtp
+ NEXTN_PROJ_PRE = auto()
+ NEXTN_PROJ_POST = auto()
+ NEXTN_EH_PROJ = auto()
+ NEXTN_EMBED_TOKENS = auto()
+ NEXTN_ENORM = auto()
+ NEXTN_HNORM = auto()
+ NEXTN_SHARED_HEAD_HEAD = auto()
+ NEXTN_SHARED_HEAD_NORM = auto()
+ NEXTN_HC_HEAD_NORM = auto()
+ NEXTN_HC_HEAD_DOWN = auto()
+ NEXTN_HC_HEAD_UP = auto()
+ # eagle3
+ FC = auto() # feature fusion layer
+ D2T = auto() # draft to target vocabulary mapping
+ # dspark
+ DSPARK_MARKOV_W1 = auto() # markov head: prev-token embed
+ DSPARK_MARKOV_W2 = auto() # markov head: bias projection
+ DSPARK_CONF_PROJ = auto() # confidence head
+ DFLASH_ATTN_CONV_BASE = auto()
+ DFLASH_ATTN_CONV_PROJ = auto()
+ DFLASH_FFN_CONV_BASE = auto()
+ DFLASH_FFN_CONV_PROJ = auto()
+ DFLASH_SELECTOR_PREV = auto()
+ DFLASH_SELECTOR_NEXT = auto()
+ DFLASH_SELECTOR_HIDDEN = auto()
+ # lfm2 audio
+ A_ENC_NORM_CONV = auto()
+ A_ENC_LINEAR_POS = auto()
+ A_ENC_POS_BIAS_U = auto()
+ A_ENC_POS_BIAS_V = auto()
+ A_ENC_OUT = auto()
+ A_ENC_CONV_DW = auto() # SSM conv
+ A_ENC_CONV_NORM = auto() # SSM conv
+ A_ENC_CONV_PW1 = auto()
+ A_ENC_CONV_PW2 = auto()
+ A_ENC_CONV_NORM_MEAN = auto() # parakeet
+ A_ENC_CONV_NORM_VAR = auto() # parakeet
+ A_ENC_MEL_FILTERS = auto() # parakeet
+ A_ENC_WINDOW = auto() # parakeet
+ A_CTC_OUT = auto()
+ A_CTC_OUT_MID = auto()
+ A_ENC_ATTN_REL_POS_EMB = auto()
+ # audio qformer projector
+ A_QF_PROJ_QUERY = auto()
+ A_QF_PROJ_NORM = auto()
+ A_QF_PROJ_LINEAR = auto()
+ A_QF_SELF_ATTN_Q = auto()
+ A_QF_SELF_ATTN_K = auto()
+ A_QF_SELF_ATTN_V = auto()
+ A_QF_SELF_ATTN_O = auto()
+ A_QF_SELF_ATTN_NORM = auto()
+ A_QF_CROSS_ATTN_Q = auto()
+ A_QF_CROSS_ATTN_K = auto()
+ A_QF_CROSS_ATTN_V = auto()
+ A_QF_CROSS_ATTN_O = auto()
+ A_QF_CROSS_ATTN_NORM = auto()
+ A_QF_FFN_UP = auto()
+ A_QF_FFN_DOWN = auto()
+ A_QF_FFN_NORM = auto()
+
+
+MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
+ MODEL_ARCH.MMPROJ: "clip", # dummy arch for clip.cpp
+ MODEL_ARCH.LLAMA: "llama",
+ MODEL_ARCH.LLAMA4: "llama4",
+ MODEL_ARCH.DECI: "deci",
+ MODEL_ARCH.FALCON: "falcon",
+ MODEL_ARCH.BAICHUAN: "baichuan",
+ MODEL_ARCH.GROK: "grok",
+ MODEL_ARCH.GPT2: "gpt2",
+ MODEL_ARCH.GPTJ: "gptj",
+ MODEL_ARCH.GPTNEOX: "gptneox",
+ MODEL_ARCH.MPT: "mpt",
+ MODEL_ARCH.STARCODER: "starcoder",
+ MODEL_ARCH.REFACT: "refact",
+ MODEL_ARCH.BERT: "bert",
+ MODEL_ARCH.MODERN_BERT: "modern-bert",
+ MODEL_ARCH.NOMIC_BERT: "nomic-bert",
+ MODEL_ARCH.NOMIC_BERT_MOE: "nomic-bert-moe",
+ MODEL_ARCH.NEO_BERT: "neo-bert",
+ MODEL_ARCH.JINA_BERT_V2: "jina-bert-v2",
+ MODEL_ARCH.JINA_BERT_V3: "jina-bert-v3",
+ MODEL_ARCH.EUROBERT: "eurobert",
+ MODEL_ARCH.BLOOM: "bloom",
+ MODEL_ARCH.STABLELM: "stablelm",
+ MODEL_ARCH.QWEN: "qwen",
+ MODEL_ARCH.QWEN2: "qwen2",
+ MODEL_ARCH.QWEN2MOE: "qwen2moe",
+ MODEL_ARCH.QWEN2VL: "qwen2vl",
+ MODEL_ARCH.QWEN3: "qwen3",
+ MODEL_ARCH.QWEN3MOE: "qwen3moe",
+ MODEL_ARCH.QWEN3NEXT: "qwen3next",
+ MODEL_ARCH.HRM_TEXT: "hrm_text",
+ MODEL_ARCH.QWEN3VL: "qwen3vl",
+ MODEL_ARCH.QWEN3VLMOE: "qwen3vlmoe",
+ MODEL_ARCH.QWEN35: "qwen35",
+ MODEL_ARCH.QWEN35MOE: "qwen35moe",
+ MODEL_ARCH.CLEF: "clef",
+ MODEL_ARCH.VLLM_DECISION2: "vllm-decision2",
+ MODEL_ARCH.QWEN4EXP: "qwen4exp",
+ MODEL_ARCH.PHI2: "phi2",
+ MODEL_ARCH.PHI3: "phi3",
+ MODEL_ARCH.PHIMOE: "phimoe",
+ MODEL_ARCH.PLAMO: "plamo",
+ MODEL_ARCH.PLAMO2: "plamo2",
+ MODEL_ARCH.PLAMO3: "plamo3",
+ MODEL_ARCH.CODESHELL: "codeshell",
+ MODEL_ARCH.ORION: "orion",
+ MODEL_ARCH.INTERNLM2: "internlm2",
+ MODEL_ARCH.MINICPM: "minicpm",
+ MODEL_ARCH.MINICPM3: "minicpm3",
+ MODEL_ARCH.GEMMA: "gemma",
+ MODEL_ARCH.GEMMA2: "gemma2",
+ MODEL_ARCH.GEMMA3: "gemma3",
+ MODEL_ARCH.GEMMA3N: "gemma3n",
+ MODEL_ARCH.GEMMA4: "gemma4",
+ MODEL_ARCH.GEMMA4_ASSISTANT: "gemma4-assistant",
+ MODEL_ARCH.GEMMA_EMBEDDING: "gemma-embedding",
+ MODEL_ARCH.GEMMA_EMBEDDING2: "gemma-embedding2",
+ MODEL_ARCH.STARCODER2: "starcoder2",
+ MODEL_ARCH.RWKV6: "rwkv6",
+ MODEL_ARCH.RWKV6QWEN2: "rwkv6qwen2",
+ MODEL_ARCH.RWKV7: "rwkv7",
+ MODEL_ARCH.ARWKV7: "arwkv7",
+ MODEL_ARCH.MAMBA: "mamba",
+ MODEL_ARCH.MAMBA2: "mamba2",
+ MODEL_ARCH.MAPLE: "maple",
+ MODEL_ARCH.JAMBA: "jamba",
+ MODEL_ARCH.XVERSE: "xverse",
+ MODEL_ARCH.COMMAND_R: "command-r",
+ MODEL_ARCH.COHERE2: "cohere2",
+ MODEL_ARCH.COHERE2MOE: "cohere2moe",
+ MODEL_ARCH.DBRX: "dbrx",
+ MODEL_ARCH.OLMO: "olmo",
+ MODEL_ARCH.OLMO2: "olmo2",
+ MODEL_ARCH.OLMOE: "olmoe",
+ MODEL_ARCH.MUSE_GLIMMER: "muse-glimmer",
+ MODEL_ARCH.OPENELM: "openelm",
+ MODEL_ARCH.ARCTIC: "arctic",
+ MODEL_ARCH.DEEPSEEK: "deepseek",
+ MODEL_ARCH.DEEPSEEK2: "deepseek2",
+ MODEL_ARCH.DEEPSEEK2OCR: "deepseek2-ocr",
+ MODEL_ARCH.DEEPSEEK32: "deepseek32",
+ MODEL_ARCH.DEEPSEEK4: "deepseek4",
+ MODEL_ARCH.CHATGLM: "chatglm",
+ MODEL_ARCH.GLM4: "glm4",
+ MODEL_ARCH.GLM4_MOE: "glm4moe",
+ MODEL_ARCH.GLM_DSA: "glm-dsa",
+ MODEL_ARCH.GLM5_NEXT: "glm5-next",
+ MODEL_ARCH.BITNET: "bitnet",
+ MODEL_ARCH.T5: "t5",
+ MODEL_ARCH.T5ENCODER: "t5encoder",
+ MODEL_ARCH.JAIS: "jais",
+ MODEL_ARCH.JAIS2: "jais2",
+ MODEL_ARCH.NEMOTRON: "nemotron",
+ MODEL_ARCH.NEMOTRON_H: "nemotron_h",
+ MODEL_ARCH.NEMOTRON_H_MOE: "nemotron_h_moe",
+ MODEL_ARCH.EXAONE: "exaone",
+ MODEL_ARCH.EXAONE4: "exaone4",
+ MODEL_ARCH.EXAONE_MOE: "exaone-moe",
+ MODEL_ARCH.GRANITE: "granite",
+ MODEL_ARCH.GRANITE_MOE: "granitemoe",
+ MODEL_ARCH.GRANITE_HYBRID: "granitehybrid",
+ MODEL_ARCH.GRANITE_SWITCH: "graniteswitch",
+ MODEL_ARCH.GRANITE_SWA: "granite_swa",
+ MODEL_ARCH.CHAMELEON: "chameleon",
+ MODEL_ARCH.WAVTOKENIZER_DEC: "wavtokenizer-dec",
+ MODEL_ARCH.PLM: "plm",
+ MODEL_ARCH.BAILINGMOE: "bailingmoe",
+ MODEL_ARCH.BAILINGMOE2: "bailingmoe2",
+ MODEL_ARCH.BAILINGMOE3: "bailingmoe3",
+ MODEL_ARCH.DOTS1: "dots1",
+ MODEL_ARCH.DOTS3NOTE: "dots3note",
+ MODEL_ARCH.ARCEE: "arcee",
+ MODEL_ARCH.AFMOE: "afmoe",
+ MODEL_ARCH.LAGUNA: "laguna",
+ MODEL_ARCH.ERNIE4_5: "ernie4_5",
+ MODEL_ARCH.ERNIE4_5_MOE: "ernie4_5-moe",
+ MODEL_ARCH.FALCON_H1: "falcon-h1",
+ MODEL_ARCH.HUNYUAN_MOE: "hunyuan-moe",
+ MODEL_ARCH.HUNYUAN_DENSE: "hunyuan-dense",
+ MODEL_ARCH.HUNYUAN_VL: "hunyuan_vl",
+ MODEL_ARCH.HY_V3: "hy_v3",
+ MODEL_ARCH.HY_V4: "hy_v4",
+ MODEL_ARCH.SMOLLM3: "smollm3",
+ MODEL_ARCH.GPT_OSS: "gpt-oss",
+ MODEL_ARCH.LFM2: "lfm2",
+ MODEL_ARCH.LFM2MOE: "lfm2moe",
+ MODEL_ARCH.DREAM: "dream",
+ MODEL_ARCH.SMALLTHINKER: "smallthinker",
+ MODEL_ARCH.LLADA: "llada",
+ MODEL_ARCH.LLADA_MOE: "llada-moe",
+ MODEL_ARCH.SEED_OSS: "seed_oss",
+ MODEL_ARCH.GROVEMOE: "grovemoe",
+ MODEL_ARCH.APERTUS: "apertus",
+ MODEL_ARCH.MINIMAX01: "minimax-01",
+ MODEL_ARCH.MINIMAXM2: "minimax-m2",
+ MODEL_ARCH.MINIMAXM3: "minimax-m3",
+ MODEL_ARCH.COGVLM: "cogvlm",
+ MODEL_ARCH.RND1: "rnd1",
+ MODEL_ARCH.PANGU_EMBED: "pangu-embedded",
+ MODEL_ARCH.MISTRAL3: "mistral3",
+ MODEL_ARCH.EAGLE3: "eagle3",
+ MODEL_ARCH.DFLASH: "dflash",
+ MODEL_ARCH.MISTRAL4: "mistral4",
+ MODEL_ARCH.PADDLEOCR: "paddleocr",
+ MODEL_ARCH.MIMO2: "mimo2",
+ MODEL_ARCH.STEP35: "step35",
+ MODEL_ARCH.SPARK2_5: "spark2_5",
+ MODEL_ARCH.LLAMA_EMBED: "llama-embed",
+ MODEL_ARCH.MAINCODER: "maincoder",
+ MODEL_ARCH.KIMI_LINEAR: "kimi-linear",
+ MODEL_ARCH.KIMI_K3: "kimi-k3",
+ MODEL_ARCH.TALKIE: "talkie",
+ MODEL_ARCH.MELLUM: "mellum",
+ MODEL_ARCH.NANBEIGE: "nanbeige",
+ MODEL_ARCH.QWEN3TTS: "qwen3tts",
+ MODEL_ARCH.POCKETTTS: "pockettts",
+ MODEL_ARCH.K2HORIZON: "k2-horizon",
+}
+
+VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
+ VISION_PROJECTOR_TYPE.MLP: "mlp",
+ VISION_PROJECTOR_TYPE.LDP: "ldp",
+ VISION_PROJECTOR_TYPE.LDPV2: "ldpv2",
+ VISION_PROJECTOR_TYPE.RESAMPLER: "resampler",
+ VISION_PROJECTOR_TYPE.GLM_EDGE: "adapter",
+ VISION_PROJECTOR_TYPE.MERGER: "qwen2vl_merger",
+ VISION_PROJECTOR_TYPE.GEMMA3: "gemma3",
+ VISION_PROJECTOR_TYPE.QWEN3VL: "qwen3vl_merger",
+ VISION_PROJECTOR_TYPE.LING3VL: "ling3vl",
+ VISION_PROJECTOR_TYPE.STEP3VL: "step3vl",
+}
+
+TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
+ MODEL_TENSOR.TOKEN_EMBD: "token_embd",
+ MODEL_TENSOR.TOKEN_EMBD_NORM: "token_embd_norm",
+ MODEL_TENSOR.TOKEN_TYPES: "token_types",
+ MODEL_TENSOR.MASKED_EMBD_CENTROIDS: "masked_embd_centroids",
+ MODEL_TENSOR.MASKED_EMBD_ORDERING: "masked_embd_ordering",
+ MODEL_TENSOR.POS_EMBD: "position_embd",
+ MODEL_TENSOR.OUTPUT_NORM: "output_norm",
+ MODEL_TENSOR.OUTPUT: "output",
+ MODEL_TENSOR.HRM_Z_L_INIT: "hrm.z_l_init",
+ MODEL_TENSOR.DENSE_2_OUT: "dense_2", # embeddinggemma 2_Dense
+ MODEL_TENSOR.DENSE_3_OUT: "dense_3", # embeddinggemma 2_Dense
+ MODEL_TENSOR.HC_HEAD_FN: "output_hc_fn",
+ MODEL_TENSOR.HC_HEAD_BASE: "output_hc_base",
+ MODEL_TENSOR.HC_HEAD_SCALE: "output_hc_scale",
+ MODEL_TENSOR.HC_HEAD_NORM: "output_hc_norm", # qwen4exp
+ MODEL_TENSOR.HC_HEAD_DOWN: "output_hc_down", # qwen4exp
+ MODEL_TENSOR.HC_HEAD_UP: "output_hc_up", # qwen4exp
+ MODEL_TENSOR.ROPE_FREQS: "rope_freqs",
+ MODEL_TENSOR.ROPE_FACTORS_LONG: "rope_factors_long",
+ MODEL_TENSOR.ROPE_FACTORS_SHORT: "rope_factors_short",
+ MODEL_TENSOR.ATTN_NORM: "blk.{bid}.attn_norm",
+ MODEL_TENSOR.ATTN_NORM_2: "blk.{bid}.attn_norm_2",
+ MODEL_TENSOR.ATTN_QKV: "blk.{bid}.attn_qkv",
+ MODEL_TENSOR.ATTN_Q: "blk.{bid}.attn_q",
+ MODEL_TENSOR.ATTN_K: "blk.{bid}.attn_k",
+ MODEL_TENSOR.ATTN_V: "blk.{bid}.attn_v",
+ MODEL_TENSOR.ATTN_OUT: "blk.{bid}.attn_output",
+ MODEL_TENSOR.ATTN_ROT_EMBD: "blk.{bid}.attn_rot_embd",
+ MODEL_TENSOR.ATTN_SINKS: "blk.{bid}.attn_sinks",
+ MODEL_TENSOR.ATTN_GATE: "blk.{bid}.attn_gate",
+ MODEL_TENSOR.ATTN_Q_NORM: "blk.{bid}.attn_q_norm",
+ MODEL_TENSOR.ATTN_K_NORM: "blk.{bid}.attn_k_norm",
+ MODEL_TENSOR.ATTN_OUT_NORM: "blk.{bid}.attn_output_norm",
+ MODEL_TENSOR.ATTN_POST_NORM: "blk.{bid}.post_attention_norm",
+ MODEL_TENSOR.FFN_GATE_INP: "blk.{bid}.ffn_gate_inp",
+ MODEL_TENSOR.FFN_GATE_INP_SHEXP: "blk.{bid}.ffn_gate_inp_shexp",
+ MODEL_TENSOR.FFN_NORM: "blk.{bid}.ffn_norm",
+ MODEL_TENSOR.FFN_PRE_NORM: "blk.{bid}.ffn_norm",
+ MODEL_TENSOR.FFN_POST_NORM: "blk.{bid}.post_ffw_norm",
+ MODEL_TENSOR.FFN_PRE_NORM_2: "blk.{bid}.pre_ffw_norm_2", # gemma4
+ MODEL_TENSOR.FFN_POST_NORM_1: "blk.{bid}.post_ffw_norm_1", # gemma4
+ MODEL_TENSOR.FFN_POST_NORM_2: "blk.{bid}.post_ffw_norm_2", # gemma4
+ MODEL_TENSOR.FFN_GATE: "blk.{bid}.ffn_gate",
+ MODEL_TENSOR.FFN_DOWN: "blk.{bid}.ffn_down",
+ MODEL_TENSOR.FFN_UP: "blk.{bid}.ffn_up",
+ MODEL_TENSOR.FFN_GATE_SHEXP: "blk.{bid}.ffn_gate_shexp",
+ MODEL_TENSOR.FFN_DOWN_SHEXP: "blk.{bid}.ffn_down_shexp",
+ MODEL_TENSOR.FFN_UP_SHEXP: "blk.{bid}.ffn_up_shexp",
+ MODEL_TENSOR.FFN_GATE_CHEXP: "blk.{bid}.ffn_gate_chexps",
+ MODEL_TENSOR.FFN_DOWN_CHEXP: "blk.{bid}.ffn_down_chexps",
+ MODEL_TENSOR.FFN_UP_CHEXP: "blk.{bid}.ffn_up_chexps",
+ MODEL_TENSOR.FFN_ACT: "blk.{bid}.ffn",
+ MODEL_TENSOR.FFN_NORM_EXP: "blk.{bid}.ffn_norm_exps",
+ MODEL_TENSOR.FFN_GATE_EXP: "blk.{bid}.ffn_gate_exps",
+ MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down_exps",
+ MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps",
+ MODEL_TENSOR.FFN_GATE_UP_EXP: "blk.{bid}.ffn_gate_up_exps",
+ MODEL_TENSOR.FFN_EXP_PROBS_B: "blk.{bid}.exp_probs_b",
+ MODEL_TENSOR.FFN_EXP_PROBS_B_VL: "blk.{bid}.exp_probs_b_vl",
+ MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid",
+ MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super
+ MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super
+ MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm",
+ MODEL_TENSOR.LAYER_OUT_SCALE: "blk.{bid}.layer_output_scale",
+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD: "per_layer_token_embd", # gemma3n
+ MODEL_TENSOR.PER_LAYER_MODEL_PROJ: "per_layer_model_proj", # gemma3n
+ MODEL_TENSOR.PER_LAYER_PROJ_NORM: "per_layer_proj_norm", # gemma3n
+ MODEL_TENSOR.ALTUP_UNEMBD_PROJ: "altup_unembd_proj", # gemma3n
+ MODEL_TENSOR.ALTUP_PROJ: "altup_proj", # gemma3n
+ MODEL_TENSOR.PER_LAYER_INP_GATE: "blk.{bid}.inp_gate", # gemma3n
+ MODEL_TENSOR.PER_LAYER_PROJ: "blk.{bid}.proj", # gemma3n
+ MODEL_TENSOR.PER_LAYER_POST_NORM: "blk.{bid}.post_norm", # gemma3n
+ MODEL_TENSOR.ALTUP_CORRECT_COEF: "blk.{bid}.altup_correct_coef", # gemma3n
+ MODEL_TENSOR.ALTUP_CORRECT_SCALE: "blk.{bid}.altup_correct_scale", # gemma3n
+ MODEL_TENSOR.ALTUP_PREDICT_COEF: "blk.{bid}.altup_predict_coef", # gemma3n
+ MODEL_TENSOR.ALTUP_ROUTER: "blk.{bid}.altup_router", # gemma3n
+ MODEL_TENSOR.ALTUP_ROUTER_NORM: "blk.{bid}.altup_router_norm", # gemma3n
+ MODEL_TENSOR.LAUREL_L: "blk.{bid}.laurel_l", # gemma3n
+ MODEL_TENSOR.LAUREL_R: "blk.{bid}.laurel_r", # gemma3n
+ MODEL_TENSOR.LAUREL_POST_NORM: "blk.{bid}.laurel_post_norm", # gemma3n
+ MODEL_TENSOR.SSM_IN: "blk.{bid}.ssm_in",
+ MODEL_TENSOR.SSM_CONV1D: "blk.{bid}.ssm_conv1d",
+ MODEL_TENSOR.SSM_X: "blk.{bid}.ssm_x",
+ MODEL_TENSOR.SSM_DT: "blk.{bid}.ssm_dt",
+ MODEL_TENSOR.SSM_DT_NORM: "blk.{bid}.ssm_dt_norm",
+ MODEL_TENSOR.SSM_A: "blk.{bid}.ssm_a",
+ MODEL_TENSOR.SSM_B_NORM: "blk.{bid}.ssm_b_norm",
+ MODEL_TENSOR.SSM_C_NORM: "blk.{bid}.ssm_c_norm",
+ MODEL_TENSOR.SSM_D: "blk.{bid}.ssm_d",
+ MODEL_TENSOR.SSM_NORM: "blk.{bid}.ssm_norm",
+ MODEL_TENSOR.SSM_OUT: "blk.{bid}.ssm_out",
+ MODEL_TENSOR.SSM_ALPHA: "blk.{bid}.ssm_alpha", # qwen3.5
+ MODEL_TENSOR.SSM_BETA_ALPHA: "blk.{bid}.ssm_ba",
+ MODEL_TENSOR.SSM_CONV1D_Q: "blk.{bid}.ssm_conv1d_q", # Kimi Linear
+ MODEL_TENSOR.SSM_CONV1D_K: "blk.{bid}.ssm_conv1d_k", # Kimi Linear
+ MODEL_TENSOR.SSM_CONV1D_V: "blk.{bid}.ssm_conv1d_v", # Kimi Linear
+ MODEL_TENSOR.SSM_F_A: "blk.{bid}.ssm_f_a", # Kimi Linear
+ MODEL_TENSOR.SSM_F_B: "blk.{bid}.ssm_f_b", # Kimi Linear
+ MODEL_TENSOR.SSM_BETA: "blk.{bid}.ssm_beta", # Kimi Linear qwen3.5
+ MODEL_TENSOR.SSM_G_A: "blk.{bid}.ssm_g_a", # Kimi Linear
+ MODEL_TENSOR.SSM_G_B: "blk.{bid}.ssm_g_b", # Kimi Linear
+ MODEL_TENSOR.SSM_G: "blk.{bid}.ssm_g", # Kimi K3
+ MODEL_TENSOR.ATTN_RES_SCORE: "blk.{bid}.attn_res_score", # Kimi K3
+ MODEL_TENSOR.FFN_RES_SCORE: "blk.{bid}.ffn_res_score", # Kimi K3
+ MODEL_TENSOR.OUTPUT_RES_SCORE: "output_res_score", # Kimi K3
+ MODEL_TENSOR.FFN_ROUTED_DOWN: "blk.{bid}.ffn_routed_down", # Kimi K3
+ MODEL_TENSOR.FFN_ROUTED_UP: "blk.{bid}.ffn_routed_up", # Kimi K3
+ MODEL_TENSOR.FFN_ROUTED_NORM: "blk.{bid}.ffn_routed_norm", # Kimi K3
+ MODEL_TENSOR.TIME_MIX_W0: "blk.{bid}.time_mix_w0",
+ MODEL_TENSOR.TIME_MIX_W1: "blk.{bid}.time_mix_w1",
+ MODEL_TENSOR.TIME_MIX_W2: "blk.{bid}.time_mix_w2",
+ MODEL_TENSOR.TIME_MIX_A0: "blk.{bid}.time_mix_a0",
+ MODEL_TENSOR.TIME_MIX_A1: "blk.{bid}.time_mix_a1",
+ MODEL_TENSOR.TIME_MIX_A2: "blk.{bid}.time_mix_a2",
+ MODEL_TENSOR.TIME_MIX_V0: "blk.{bid}.time_mix_v0",
+ MODEL_TENSOR.TIME_MIX_V1: "blk.{bid}.time_mix_v1",
+ MODEL_TENSOR.TIME_MIX_V2: "blk.{bid}.time_mix_v2",
+ MODEL_TENSOR.TIME_MIX_G1: "blk.{bid}.time_mix_g1",
+ MODEL_TENSOR.TIME_MIX_G2: "blk.{bid}.time_mix_g2",
+ MODEL_TENSOR.TIME_MIX_K_K: "blk.{bid}.time_mix_k_k",
+ MODEL_TENSOR.TIME_MIX_K_A: "blk.{bid}.time_mix_k_a",
+ MODEL_TENSOR.TIME_MIX_R_K: "blk.{bid}.time_mix_r_k",
+ MODEL_TENSOR.TIME_MIX_LERP_X: "blk.{bid}.time_mix_lerp_x",
+ MODEL_TENSOR.TIME_MIX_LERP_K: "blk.{bid}.time_mix_lerp_k",
+ MODEL_TENSOR.TIME_MIX_LERP_V: "blk.{bid}.time_mix_lerp_v",
+ MODEL_TENSOR.TIME_MIX_LERP_R: "blk.{bid}.time_mix_lerp_r",
+ MODEL_TENSOR.TIME_MIX_LERP_G: "blk.{bid}.time_mix_lerp_g",
+ MODEL_TENSOR.TIME_MIX_LERP_FUSED: "blk.{bid}.time_mix_lerp_fused",
+ MODEL_TENSOR.TIME_MIX_LERP_W: "blk.{bid}.time_mix_lerp_w",
+ MODEL_TENSOR.TIME_MIX_FIRST: "blk.{bid}.time_mix_first",
+ MODEL_TENSOR.TIME_MIX_DECAY: "blk.{bid}.time_mix_decay",
+ MODEL_TENSOR.TIME_MIX_DECAY_W1: "blk.{bid}.time_mix_decay_w1",
+ MODEL_TENSOR.TIME_MIX_DECAY_W2: "blk.{bid}.time_mix_decay_w2",
+ MODEL_TENSOR.TIME_MIX_KEY: "blk.{bid}.time_mix_key",
+ MODEL_TENSOR.TIME_MIX_VALUE: "blk.{bid}.time_mix_value",
+ MODEL_TENSOR.TIME_MIX_RECEPTANCE: "blk.{bid}.time_mix_receptance",
+ MODEL_TENSOR.TIME_MIX_GATE: "blk.{bid}.time_mix_gate",
+ MODEL_TENSOR.TIME_MIX_LN: "blk.{bid}.time_mix_ln",
+ MODEL_TENSOR.TIME_MIX_OUTPUT: "blk.{bid}.time_mix_output",
+ MODEL_TENSOR.CHANNEL_MIX_LERP_K: "blk.{bid}.channel_mix_lerp_k",
+ MODEL_TENSOR.CHANNEL_MIX_LERP_R: "blk.{bid}.channel_mix_lerp_r",
+ MODEL_TENSOR.CHANNEL_MIX_KEY: "blk.{bid}.channel_mix_key",
+ MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE: "blk.{bid}.channel_mix_receptance",
+ MODEL_TENSOR.CHANNEL_MIX_VALUE: "blk.{bid}.channel_mix_value",
+ MODEL_TENSOR.ATTN_Q_A: "blk.{bid}.attn_q_a",
+ MODEL_TENSOR.ATTN_Q_B: "blk.{bid}.attn_q_b",
+ MODEL_TENSOR.ATTN_KV_A_MQA: "blk.{bid}.attn_kv_a_mqa",
+ MODEL_TENSOR.ATTN_KV_B: "blk.{bid}.attn_kv_b",
+ MODEL_TENSOR.ATTN_K_B: "blk.{bid}.attn_k_b",
+ MODEL_TENSOR.ATTN_V_B: "blk.{bid}.attn_v_b",
+ MODEL_TENSOR.ATTN_Q_A_NORM: "blk.{bid}.attn_q_a_norm",
+ MODEL_TENSOR.ATTN_KV_A_NORM: "blk.{bid}.attn_kv_a_norm",
+ MODEL_TENSOR.ATTN_KV: "blk.{bid}.attn_kv",
+ MODEL_TENSOR.ATTN_KV_NORM: "blk.{bid}.attn_kv_a_norm",
+ MODEL_TENSOR.ATTN_OUT_A: "blk.{bid}.attn_output_a",
+ MODEL_TENSOR.ATTN_OUT_B: "blk.{bid}.attn_output_b",
+ MODEL_TENSOR.HC_ATTN_FN: "blk.{bid}.hc_attn_fn",
+ MODEL_TENSOR.HC_ATTN_BASE: "blk.{bid}.hc_attn_base",
+ MODEL_TENSOR.HC_ATTN_SCALE: "blk.{bid}.hc_attn_scale",
+ MODEL_TENSOR.HC_FFN_FN: "blk.{bid}.hc_ffn_fn",
+ MODEL_TENSOR.HC_FFN_BASE: "blk.{bid}.hc_ffn_base",
+ MODEL_TENSOR.HC_FFN_SCALE: "blk.{bid}.hc_ffn_scale",
+ MODEL_TENSOR.HC_ATTN_NORM: "blk.{bid}.hc_attn_norm", # qwen4exp
+ MODEL_TENSOR.HC_ATTN_DOWN: "blk.{bid}.hc_attn_down", # qwen4exp
+ MODEL_TENSOR.HC_ATTN_UP: "blk.{bid}.hc_attn_up", # qwen4exp
+ MODEL_TENSOR.HC_ATTN_INJECT: "blk.{bid}.hc_attn_inject", # qwen4exp
+ MODEL_TENSOR.HC_FFN_NORM: "blk.{bid}.hc_ffn_norm", # qwen4exp
+ MODEL_TENSOR.HC_FFN_DOWN: "blk.{bid}.hc_ffn_down", # qwen4exp
+ MODEL_TENSOR.HC_FFN_UP: "blk.{bid}.hc_ffn_up", # qwen4exp
+ MODEL_TENSOR.HC_FFN_INJECT: "blk.{bid}.hc_ffn_inject", # qwen4exp
+ MODEL_TENSOR.PLE_KEY: "blk.{bid}.ple_key", # qwen4exp
+ MODEL_TENSOR.PLE_VALUE: "blk.{bid}.ple_value", # qwen4exp
+ MODEL_TENSOR.PLE_NORM_KEY: "blk.{bid}.ple_norm_key", # qwen4exp
+ MODEL_TENSOR.PLE_NORM_QUERY: "blk.{bid}.ple_norm_query", # qwen4exp
+ MODEL_TENSOR.PLE_NORM_CONV: "blk.{bid}.ple_norm_conv", # qwen4exp
+ MODEL_TENSOR.PLE_CONV1D: "blk.{bid}.ple_conv1d", # qwen4exp
+ MODEL_TENSOR.ATTN_COMPRESSOR_WKV: "blk.{bid}.attn_compressor_kv",
+ MODEL_TENSOR.ATTN_COMPRESSOR_WGATE: "blk.{bid}.attn_compressor_gate",
+ MODEL_TENSOR.ATTN_COMPRESSOR_APE: "blk.{bid}.attn_compressor_ape",
+ MODEL_TENSOR.ATTN_COMPRESSOR_NORM: "blk.{bid}.attn_compressor_norm",
+ MODEL_TENSOR.ATTN_SUB_NORM: "blk.{bid}.attn_sub_norm",
+ MODEL_TENSOR.FFN_SUB_NORM: "blk.{bid}.ffn_sub_norm",
+ MODEL_TENSOR.DEC_ATTN_NORM: "dec.blk.{bid}.attn_norm",
+ MODEL_TENSOR.DEC_ATTN_Q: "dec.blk.{bid}.attn_q",
+ MODEL_TENSOR.DEC_ATTN_K: "dec.blk.{bid}.attn_k",
+ MODEL_TENSOR.DEC_ATTN_V: "dec.blk.{bid}.attn_v",
+ MODEL_TENSOR.DEC_ATTN_OUT: "dec.blk.{bid}.attn_o",
+ MODEL_TENSOR.DEC_ATTN_REL_B: "dec.blk.{bid}.attn_rel_b",
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM: "dec.blk.{bid}.cross_attn_norm",
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV: "dec.blk.{bid}.cross_attn_norm_kv",
+ MODEL_TENSOR.DEC_CROSS_ATTN_Q: "dec.blk.{bid}.cross_attn_q",
+ MODEL_TENSOR.DEC_CROSS_ATTN_K: "dec.blk.{bid}.cross_attn_k",
+ MODEL_TENSOR.DEC_CROSS_ATTN_V: "dec.blk.{bid}.cross_attn_v",
+ MODEL_TENSOR.DEC_CROSS_ATTN_OUT: "dec.blk.{bid}.cross_attn_o",
+ MODEL_TENSOR.DEC_CROSS_ATTN_REL_B: "dec.blk.{bid}.cross_attn_rel_b",
+ MODEL_TENSOR.DEC_FFN_NORM: "dec.blk.{bid}.ffn_norm",
+ MODEL_TENSOR.DEC_FFN_GATE: "dec.blk.{bid}.ffn_gate",
+ MODEL_TENSOR.DEC_FFN_DOWN: "dec.blk.{bid}.ffn_down",
+ MODEL_TENSOR.DEC_FFN_UP: "dec.blk.{bid}.ffn_up",
+ MODEL_TENSOR.DEC_OUTPUT_NORM: "dec.output_norm",
+ MODEL_TENSOR.ENC_ATTN_NORM: "enc.blk.{bid}.attn_norm",
+ MODEL_TENSOR.ENC_ATTN_Q: "enc.blk.{bid}.attn_q",
+ MODEL_TENSOR.ENC_ATTN_K: "enc.blk.{bid}.attn_k",
+ MODEL_TENSOR.ENC_ATTN_V: "enc.blk.{bid}.attn_v",
+ MODEL_TENSOR.ENC_ATTN_OUT: "enc.blk.{bid}.attn_o",
+ MODEL_TENSOR.ENC_ATTN_REL_B: "enc.blk.{bid}.attn_rel_b",
+ MODEL_TENSOR.ENC_FFN_NORM: "enc.blk.{bid}.ffn_norm",
+ MODEL_TENSOR.ENC_FFN_GATE: "enc.blk.{bid}.ffn_gate",
+ MODEL_TENSOR.ENC_FFN_DOWN: "enc.blk.{bid}.ffn_down",
+ MODEL_TENSOR.ENC_FFN_UP: "enc.blk.{bid}.ffn_up",
+ MODEL_TENSOR.ENC_OUTPUT_NORM: "enc.output_norm",
+ MODEL_TENSOR.CLS: "cls",
+ MODEL_TENSOR.CLS_OUT: "cls.output",
+ MODEL_TENSOR.CLS_NORM: "cls.norm",
+ MODEL_TENSOR.DECISION_HIDDEN_NORM: "decision.hidden_norm",
+ MODEL_TENSOR.DECISION_PROJ_MEMORY: "decision.proj_memory",
+ MODEL_TENSOR.DECISION_PROJ_QUESTION: "decision.proj_question",
+ MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION: "decision.proj_option_question",
+ MODEL_TENSOR.DECISION_PROJ_GLOBAL: "decision.proj_global",
+ MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT: "decision.proj_option_context",
+ MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL: "decision.proj_option_lexical",
+ MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM: "decision.option_summary_norm",
+ MODEL_TENSOR.DECISION_FIELD_NORM: "decision.field_norm",
+ MODEL_TENSOR.DECISION_OPTION_NORM: "decision.option_norm",
+ MODEL_TENSOR.DECISION_SCALES: "decision.scales",
+ MODEL_TENSOR.DECISION_SCORER: "decision.scorer",
+ MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out",
+ MODEL_TENSOR.DECISION_QUERY_NORM: "decision.query_norm",
+ MODEL_TENSOR.DECISION_KEY: "decision.key",
+ MODEL_TENSOR.DECISION_QUERY: "decision.query",
+ MODEL_TENSOR.DECISION_OPTION_MLP: "decision.option_mlp",
+ MODEL_TENSOR.DECISION_QUERY_MLP: "decision.query_mlp",
+ MODEL_TENSOR.CONV1D: "conv1d",
+ MODEL_TENSOR.CONVNEXT_DW: "convnext.{bid}.dw",
+ MODEL_TENSOR.CONVNEXT_NORM: "convnext.{bid}.norm",
+ MODEL_TENSOR.CONVNEXT_PW1: "convnext.{bid}.pw1",
+ MODEL_TENSOR.CONVNEXT_PW2: "convnext.{bid}.pw2",
+ MODEL_TENSOR.CONVNEXT_GAMMA: "convnext.{bid}.gamma",
+ MODEL_TENSOR.POSNET_CONV1: "posnet.{bid}.conv1",
+ MODEL_TENSOR.POSNET_CONV2: "posnet.{bid}.conv2",
+ MODEL_TENSOR.POSNET_NORM: "posnet.{bid}.norm",
+ MODEL_TENSOR.POSNET_NORM1: "posnet.{bid}.norm1",
+ MODEL_TENSOR.POSNET_NORM2: "posnet.{bid}.norm2",
+ MODEL_TENSOR.POSNET_ATTN_NORM: "posnet.{bid}.attn_norm",
+ MODEL_TENSOR.POSNET_ATTN_Q: "posnet.{bid}.attn_q",
+ MODEL_TENSOR.POSNET_ATTN_K: "posnet.{bid}.attn_k",
+ MODEL_TENSOR.POSNET_ATTN_V: "posnet.{bid}.attn_v",
+ MODEL_TENSOR.POSNET_ATTN_OUT: "posnet.{bid}.attn_output",
+ MODEL_TENSOR.SHORTCONV_CONV: "blk.{bid}.shortconv.conv",
+ MODEL_TENSOR.SHORTCONV_INPROJ: "blk.{bid}.shortconv.in_proj",
+ MODEL_TENSOR.SHORTCONV_OUTPROJ: "blk.{bid}.shortconv.out_proj",
+ MODEL_TENSOR.VISEXP_ATTN_QKV: "blk.{bid}.vis_attn_qkv",
+ MODEL_TENSOR.VISEXP_ATTN_OUT: "blk.{bid}.vis_attn_output",
+ MODEL_TENSOR.VISEXP_GATE: "blk.{bid}.vis_gate",
+ MODEL_TENSOR.VISEXP_DOWN: "blk.{bid}.vis_down",
+ MODEL_TENSOR.VISEXP_UP: "blk.{bid}.vis_up",
+ MODEL_TENSOR.INDEXER_K_NORM: "blk.{bid}.indexer.k_norm",
+ MODEL_TENSOR.INDEXER_PROJ: "blk.{bid}.indexer.proj",
+ MODEL_TENSOR.INDEXER_ATTN_K: "blk.{bid}.indexer.attn_k",
+ MODEL_TENSOR.INDEXER_ATTN_Q_B: "blk.{bid}.indexer.attn_q_b",
+ MODEL_TENSOR.INDEXER_Q_PROJ: "blk.{bid}.indexer.q_proj",
+ MODEL_TENSOR.INDEXER_K_PROJ: "blk.{bid}.indexer.k_proj",
+ MODEL_TENSOR.INDEXER_Q_NORM: "blk.{bid}.indexer.q_norm",
+ MODEL_TENSOR.INDEXER_COMPRESSOR_WKV: "blk.{bid}.indexer_compressor_kv",
+ MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE: "blk.{bid}.indexer_compressor_gate",
+ MODEL_TENSOR.INDEXER_COMPRESSOR_APE: "blk.{bid}.indexer_compressor_ape",
+ MODEL_TENSOR.INDEXER_COMPRESSOR_NORM: "blk.{bid}.indexer_compressor_norm",
+ MODEL_TENSOR.INDEXER_KPOOL_GATE: "blk.{bid}.indexer_compressor_gate",
+ MODEL_TENSOR.INDEXER_KPOOL_APE: "blk.{bid}.indexer_compressor_ape",
+ # vision
+ MODEL_TENSOR.V_MMPROJ: "mm.{bid}",
+ MODEL_TENSOR.V_MMPROJ_FC: "mm.model.fc",
+ MODEL_TENSOR.V_MMPROJ_MLP: "mm.model.mlp.{bid}",
+ MODEL_TENSOR.V_MMPROJ_PEG: "mm.model.peg.{bid}",
+ MODEL_TENSOR.V_ENC_EMBD_CLS: "v.class_embd",
+ MODEL_TENSOR.V_ENC_EMBD_PATCH: "v.patch_embd",
+ MODEL_TENSOR.V_ENC_EMBD_NORM: "v.norm_embd",
+ MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: "v.patch_norm.{bid}",
+ MODEL_TENSOR.V_ENC_EMBD_POS: "v.position_embd",
+ MODEL_TENSOR.V_ENC_ATTN_QKV: "v.blk.{bid}.attn_qkv",
+ MODEL_TENSOR.V_ENC_ATTN_Q: "v.blk.{bid}.attn_q",
+ MODEL_TENSOR.V_ENC_ATTN_Q_NORM: "v.blk.{bid}.attn_q_norm",
+ MODEL_TENSOR.V_ENC_ATTN_K: "v.blk.{bid}.attn_k",
+ MODEL_TENSOR.V_ENC_ATTN_K_NORM: "v.blk.{bid}.attn_k_norm",
+ MODEL_TENSOR.V_ENC_ATTN_V: "v.blk.{bid}.attn_v",
+ MODEL_TENSOR.V_ENC_INPUT_NORM: "v.blk.{bid}.ln1",
+ MODEL_TENSOR.V_ENC_ATTN_O: "v.blk.{bid}.attn_out",
+ MODEL_TENSOR.V_ENC_ATTN_O_NORM: "v.blk.{bid}.attn_out_norm",
+ MODEL_TENSOR.V_ENC_ATTN_SINKS: "v.blk.{bid}.attn_sinks",
+ MODEL_TENSOR.V_ENC_POST_ATTN_NORM: "v.blk.{bid}.ln2",
+ MODEL_TENSOR.V_ENC_FFN_UP: "v.blk.{bid}.ffn_up",
+ MODEL_TENSOR.V_ENC_FFN_GATE: "v.blk.{bid}.ffn_gate",
+ MODEL_TENSOR.V_ENC_FFN_DOWN: "v.blk.{bid}.ffn_down",
+ MODEL_TENSOR.V_ENC_FFN_GATE_INP: "v.blk.{bid}.ffn_gate_inp",
+ MODEL_TENSOR.V_ENC_FFN_GATE_EXPS: "v.blk.{bid}.ffn_gate_exps",
+ MODEL_TENSOR.V_ENC_FFN_UP_EXPS: "v.blk.{bid}.ffn_up_exps",
+ MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS: "v.blk.{bid}.ffn_down_exps",
+ MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B: "v.blk.{bid}.exp_probs_b",
+ MODEL_TENSOR.V_ENC_ATTN_POST_NORM: "v.blk.{bid}.attn_post_norm",
+ MODEL_TENSOR.V_ENC_FFN_POST_NORM: "v.blk.{bid}.ffn_post_norm",
+ MODEL_TENSOR.V_LAYER_SCALE_1: "v.blk.{bid}.ls1",
+ MODEL_TENSOR.V_LAYER_SCALE_2: "v.blk.{bid}.ls2",
+ MODEL_TENSOR.V_LAYER_OUT_SCALE: "v.blk.{bid}.out_scale",
+ MODEL_TENSOR.V_PRE_NORM: "v.pre_ln",
+ MODEL_TENSOR.V_POST_NORM: "v.post_ln",
+ MODEL_TENSOR.V_MM_POST_NORM: "mm.post_norm",
+ MODEL_TENSOR.V_MM_INP_PROJ: "mm.input_projection",
+ MODEL_TENSOR.V_MM_INP_NORM: "mm.input_norm",
+ MODEL_TENSOR.V_MM_SOFT_EMB_NORM: "mm.soft_emb_norm", # gemma3n
+ MODEL_TENSOR.V_MM_EMBEDDING: "mm.embedding", # gemma3n
+ MODEL_TENSOR.V_MM_HARD_EMB_NORM: "mm.hard_emb_norm", # gemma3n
+ MODEL_TENSOR.V_ENC_CONV_STEM: "v.conv_stem.conv", # gemma3n
+ MODEL_TENSOR.V_ENC_CONV_STEM_NORM: "v.conv_stem.bn", # gemma3n
+ MODEL_TENSOR.V_ENC_MSFA_EXP: "v.msfa.ffn.pw_exp.conv", # gemma3n
+ MODEL_TENSOR.V_ENC_MSFA_EXP_NORM: "v.msfa.ffn.pw_exp.bn", # gemma3n
+ MODEL_TENSOR.V_ENC_MSFA_PROJ: "v.msfa.ffn.pw_proj.conv", # gemma3n
+ MODEL_TENSOR.V_ENC_MSFA_PROJ_NORM: "v.msfa.ffn.pw_proj.bn", # gemma3n
+ MODEL_TENSOR.V_ENC_MSFA_NORM: "v.msfa.norm", # gemma3n
+ MODEL_TENSOR.V_RESMPL_POS_EMBD_K: "resampler.pos_embd_k",
+ MODEL_TENSOR.V_RESMPL_ATTN_Q: "resampler.attn.q",
+ MODEL_TENSOR.V_RESMPL_ATTN_K: "resampler.attn.k",
+ MODEL_TENSOR.V_RESMPL_ATTN_V: "resampler.attn.v",
+ MODEL_TENSOR.V_RESMPL_ATTN_OUT: "resampler.attn.out",
+ MODEL_TENSOR.V_RESMPL_KV: "resampler.kv",
+ MODEL_TENSOR.V_RESMPL_KV_NORM: "resampler.ln_kv",
+ MODEL_TENSOR.V_RESMPL_POST_NORM: "resampler.ln_post",
+ MODEL_TENSOR.V_RESMPL_Q_NORM: "resampler.ln_q",
+ MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj",
+ MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query",
+ MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral
+ MODEL_TENSOR.V_TOK_EMBD_IMG_START: "v.token_embd.img_start", # deepseek4v
+ MODEL_TENSOR.V_TOK_EMBD_IMG_END: "v.token_embd.img_end", # deepseek4v
+ MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: "v.token_embd.img_pad", # deepseek4v
+ MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1
+ MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm",
+ MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1",
+ MODEL_TENSOR.V_DS_FC2: "v.deepstack.{bid}.fc2",
+ MODEL_TENSOR.V_MERGER_LN1: "v.vit_merger.ln1",
+ MODEL_TENSOR.V_MERGER_ATTN_Q: "v.vit_merger.attn_q",
+ MODEL_TENSOR.V_MERGER_ATTN_K: "v.vit_merger.attn_k",
+ MODEL_TENSOR.V_MERGER_ATTN_V: "v.vit_merger.attn_v",
+ MODEL_TENSOR.V_MERGER_ATTN_O: "v.vit_merger.attn_out",
+ MODEL_TENSOR.V_MERGER_DS_LN: "v.vit_merger.ds_ln",
+ MODEL_TENSOR.V_MERGER_DS_UP: "v.vit_merger.ds_ffn_up",
+ MODEL_TENSOR.V_MERGER_DS_DOWN: "v.vit_merger.ds_ffn_down",
+ MODEL_TENSOR.V_MM_POST_FC_NORM: "mm.post_fc_norm", # cogvlm
+ MODEL_TENSOR.V_MM_UP: "mm.up",
+ MODEL_TENSOR.V_MM_DOWN: "mm.down",
+ MODEL_TENSOR.V_MM_GATE: "mm.gate",
+ MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1",
+ MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2",
+ MODEL_TENSOR.V_TOK_BOI: "v.boi",
+ MODEL_TENSOR.V_TOK_EOI: "v.eoi",
+ MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm",
+ MODEL_TENSOR.V_TOK_IMG_BEGIN: "mm.image_begin",
+ MODEL_TENSOR.V_TOK_IMG_END: "mm.image_end",
+ MODEL_TENSOR.V_STD_BIAS: "v.std_bias", # gemma4
+ MODEL_TENSOR.V_STD_SCALE: "v.std_scale", # gemma4
+ # DeepSeek-OCR SAM
+ MODEL_TENSOR.V_SAM_POS_EMBD: "v.sam.pos_embd",
+ MODEL_TENSOR.V_SAM_PATCH_EMBD: "v.sam.patch_embd",
+ MODEL_TENSOR.V_SAM_PRE_NORM: "v.sam.blk.{bid}.pre_ln",
+ MODEL_TENSOR.V_SAM_POST_NORM: "v.sam.blk.{bid}.post_ln",
+ MODEL_TENSOR.V_SAM_ATTN_POS_H: "v.sam.blk.{bid}.attn.pos_h",
+ MODEL_TENSOR.V_SAM_ATTN_POS_W: "v.sam.blk.{bid}.attn.pos_w",
+ MODEL_TENSOR.V_SAM_ATTN_QKV: "v.sam.blk.{bid}.attn.qkv",
+ MODEL_TENSOR.V_SAM_ATTN_OUT: "v.sam.blk.{bid}.attn.out",
+ MODEL_TENSOR.V_SAM_MLP_LIN_1: "v.sam.blk.{bid}.mlp.lin1",
+ MODEL_TENSOR.V_SAM_MLP_LIN_2: "v.sam.blk.{bid}.mlp.lin2",
+ MODEL_TENSOR.V_SAM_NECK: "v.sam.neck.{bid}",
+ MODEL_TENSOR.V_SAM_NET_2: "v.sam.net_2",
+ MODEL_TENSOR.V_SAM_NET_3: "v.sam.net_3",
+ MODEL_TENSOR.V_ENC_EMBD_IMGNL: "v.image_newline", # Deepseek-OCR, Granite4Vision
+ MODEL_TENSOR.V_ENC_EMBD_VSEP: "v.view_seperator", # Deepseek-OCR
+ MODEL_TENSOR.V_TOK_EMBD_SEP: "v.tok_embd_sep", # MiniCPM-V 4.7
+ MODEL_TENSOR.V_RESMPL_QUERY_768: "v.resample_query_768", # Deepseek-OCR-2 qwen2
+ MODEL_TENSOR.V_RESMPL_QUERY_1024: "v.resample_query_1024", # Deepseek-OCR-2 qwen2
+ # Granite4 Vision
+ # qformer layers (bid => proj_id)
+ # NOTE: Names align with A_QF_*
+ MODEL_TENSOR.V_QF_SELF_ATTN_Q: "v.proj_blk.{bid}.self_attn_q",
+ MODEL_TENSOR.V_QF_SELF_ATTN_K: "v.proj_blk.{bid}.self_attn_k",
+ MODEL_TENSOR.V_QF_SELF_ATTN_V: "v.proj_blk.{bid}.self_attn_v",
+ MODEL_TENSOR.V_QF_SELF_ATTN_O: "v.proj_blk.{bid}.self_attn_out",
+ MODEL_TENSOR.V_QF_SELF_ATTN_NORM: "v.proj_blk.{bid}.self_attn_norm",
+ MODEL_TENSOR.V_QF_CROSS_ATTN_Q: "v.proj_blk.{bid}.cross_attn_q",
+ MODEL_TENSOR.V_QF_CROSS_ATTN_K: "v.proj_blk.{bid}.cross_attn_k",
+ MODEL_TENSOR.V_QF_CROSS_ATTN_V: "v.proj_blk.{bid}.cross_attn_v",
+ MODEL_TENSOR.V_QF_CROSS_ATTN_O: "v.proj_blk.{bid}.cross_attn_out",
+ MODEL_TENSOR.V_QF_CROSS_ATTN_NORM: "v.proj_blk.{bid}.cross_attn_norm",
+ MODEL_TENSOR.V_QF_FFN_UP: "v.proj_blk.{bid}.ffn_up",
+ MODEL_TENSOR.V_QF_FFN_DOWN: "v.proj_blk.{bid}.ffn_down",
+ MODEL_TENSOR.V_QF_FFN_NORM: "v.proj_blk.{bid}.ffn_norm",
+ # multi-projector (bid => projector ID)
+ MODEL_TENSOR.V_MULTI_PROJ_IMG_POS: "v.proj_blk.{bid}.img_pos",
+ MODEL_TENSOR.V_MULTI_PROJ_QUERY: "v.proj_blk.{bid}.query",
+ MODEL_TENSOR.V_MULTI_PROJ_NORM: "v.proj_blk.{bid}.norm",
+ MODEL_TENSOR.V_MULTI_PROJ_LINEAR: "v.proj_blk.{bid}.linear",
+ MODEL_TENSOR.V_MULTI_PROJ_POST_NORM: "v.proj_blk.{bid}.post_norm",
+
+ # audio (mtmd)
+ # note: all audio tensor names must use prefix "a." or "mm.a."
+ MODEL_TENSOR.A_ENC_EMBD_POS: "a.position_embd",
+ MODEL_TENSOR.A_ENC_EMBD_NORM: "a.position_embd_norm",
+ MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS: "a.embd_to_logits",
+ MODEL_TENSOR.A_ENC_INP_PROJ: "a.input_projection",
+ MODEL_TENSOR.A_ENC_CONV1D: "a.conv1d.{bid}",
+ MODEL_TENSOR.A_ENC_CONV2D: "a.conv2d.{bid}",
+ MODEL_TENSOR.A_ENC_CONV_OUT: "a.conv_out",
+ MODEL_TENSOR.A_ENC_CONV1D_NORM: "a.conv1d.{bid}.norm",
+ MODEL_TENSOR.A_PRE_NORM: "a.pre_ln",
+ MODEL_TENSOR.A_POST_NORM: "a.post_ln",
+ MODEL_TENSOR.A_ENC_LAYER_PRE_NORM: "a.blk.{bid}.layer_pre_norm",
+ MODEL_TENSOR.A_ENC_ATTN_Q: "a.blk.{bid}.attn_q",
+ MODEL_TENSOR.A_ENC_ATTN_K: "a.blk.{bid}.attn_k",
+ MODEL_TENSOR.A_ENC_ATTN_V: "a.blk.{bid}.attn_v",
+ MODEL_TENSOR.A_ENC_ATTN_POST_NORM: "a.blk.{bid}.attn_post_norm",
+ MODEL_TENSOR.A_ENC_ATTN_PRE_NORM: "a.blk.{bid}.attn_pre_norm",
+ MODEL_TENSOR.A_ENC_ATTN_K_REL: "a.blk.{bid}.attn_k_rel",
+ MODEL_TENSOR.A_ENC_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale",
+ MODEL_TENSOR.A_ENC_INPUT_NORM: "a.blk.{bid}.ln1",
+ MODEL_TENSOR.A_ENC_OUTPUT: "a.blk.{bid}.attn_out",
+ MODEL_TENSOR.A_ENC_OUTPUT_NORM: "a.blk.{bid}.ln2",
+ MODEL_TENSOR.A_ENC_FFN_NORM: "a.blk.{bid}.ffn_norm",
+ MODEL_TENSOR.A_ENC_FFN_POST_NORM: "a.blk.{bid}.ffn_post_norm",
+ MODEL_TENSOR.A_ENC_FFN_SCALE: "a.blk.{bid}.ffn_scale",
+ MODEL_TENSOR.A_ENC_FFN_UP: "a.blk.{bid}.ffn_up",
+ MODEL_TENSOR.A_ENC_FFN_GATE: "a.blk.{bid}.ffn_gate",
+ MODEL_TENSOR.A_ENC_FFN_DOWN: "a.blk.{bid}.ffn_down",
+ MODEL_TENSOR.A_ENC_FFN_NORM_1: "a.blk.{bid}.ffn_norm_1",
+ MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: "a.blk.{bid}.ffn_post_norm_1",
+ MODEL_TENSOR.A_ENC_FFN_SCALE_1: "a.blk.{bid}.ffn_scale_1",
+ MODEL_TENSOR.A_ENC_FFN_UP_1: "a.blk.{bid}.ffn_up_1",
+ MODEL_TENSOR.A_ENC_FFN_GATE_1: "a.blk.{bid}.ffn_gate_1",
+ MODEL_TENSOR.A_ENC_FFN_DOWN_1: "a.blk.{bid}.ffn_down_1",
+ MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: "a.downsample.conv",
+ MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: "a.downsample.norm",
+ MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: "a.rvq.codebook",
+ MODEL_TENSOR.A_ENC_CONV_RES2: "a.blk.{bid}.res2.{xid}",
+ MODEL_TENSOR.A_ENC_SE_CONV1: "a.blk.{bid}.se_conv1",
+ MODEL_TENSOR.A_ENC_SE_CONV2: "a.blk.{bid}.se_conv2",
+ MODEL_TENSOR.A_ENC_ASP_ATTN: "a.asp_attn",
+ MODEL_TENSOR.A_ENC_ASP_TDNN: "a.asp_tdnn",
+ MODEL_TENSOR.A_GEN_CODE_PROJ_IN: "a.gen.code.proj_in",
+ MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd",
+ MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head",
+ MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd",
+ MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1)
+ MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q",
+ MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm",
+ MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k",
+ MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm",
+ MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v",
+ MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out",
+ MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2)
+ MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate",
+ MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up",
+ MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down",
+ MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM: "a.gen.code.output_norm",
+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj",
+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook",
+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_IN: "a.gen.wav.quant.rest.in_proj",
+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_OUT: "a.gen.wav.quant.rest.out_proj",
+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_CB: "a.gen.wav.quant.rest.codebook",
+ MODEL_TENSOR.A_GEN_WAV_PRE_CONV: "a.gen.wav.pre_conv",
+ MODEL_TENSOR.A_GEN_WAV_TFM_IN_PROJ: "a.gen.wav.tfm.in_proj",
+ MODEL_TENSOR.A_GEN_WAV_TFM_OUT_PROJ: "a.gen.wav.tfm.out_proj",
+ MODEL_TENSOR.A_GEN_WAV_TFM_OUTPUT_NORM: "a.gen.wav.tfm.output_norm",
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM: "a.gen.wav.tfm.blk.{bid}.ln1",
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q: "a.gen.wav.tfm.blk.{bid}.attn_q",
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K: "a.gen.wav.tfm.blk.{bid}.attn_k",
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V: "a.gen.wav.tfm.blk.{bid}.attn_v",
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT: "a.gen.wav.tfm.blk.{bid}.attn_out",
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE: "a.gen.wav.tfm.blk.{bid}.ls1",
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM: "a.gen.wav.tfm.blk.{bid}.ln2",
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE: "a.gen.wav.tfm.blk.{bid}.ffn_gate",
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP: "a.gen.wav.tfm.blk.{bid}.ffn_up",
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN: "a.gen.wav.tfm.blk.{bid}.ffn_down",
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE: "a.gen.wav.tfm.blk.{bid}.ls2",
+ MODEL_TENSOR.A_GEN_WAV_UP_CONV: "a.gen.wav.up.blk.{bid}.conv",
+ MODEL_TENSOR.A_GEN_WAV_UP_DWCONV: "a.gen.wav.up.blk.{bid}.dwconv",
+ MODEL_TENSOR.A_GEN_WAV_UP_NORM: "a.gen.wav.up.blk.{bid}.norm",
+ MODEL_TENSOR.A_GEN_WAV_UP_PW1: "a.gen.wav.up.blk.{bid}.pw1",
+ MODEL_TENSOR.A_GEN_WAV_UP_PW2: "a.gen.wav.up.blk.{bid}.pw2",
+ MODEL_TENSOR.A_GEN_WAV_UP_GAMMA: "a.gen.wav.up.blk.{bid}.gamma",
+ MODEL_TENSOR.A_GEN_WAV_DAC_ENTRY: "a.gen.wav.dac.entry",
+ MODEL_TENSOR.A_GEN_WAV_DAC_UP_SNAKE: "a.gen.wav.dac.blk.{bid}.snake",
+ MODEL_TENSOR.A_GEN_WAV_DAC_UP_CONV: "a.gen.wav.dac.blk.{bid}.conv",
+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT1: "a.gen.wav.dac.blk.{bid}.res.{xid}.act1",
+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV1: "a.gen.wav.dac.blk.{bid}.res.{xid}.conv1",
+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT2: "a.gen.wav.dac.blk.{bid}.res.{xid}.act2",
+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV2: "a.gen.wav.dac.blk.{bid}.res.{xid}.conv2",
+ MODEL_TENSOR.A_GEN_WAV_DAC_POST_SNAKE: "a.gen.wav.dac.post_snake",
+ MODEL_TENSOR.A_GEN_WAV_DAC_POST_CONV: "a.gen.wav.dac.post_conv",
+ MODEL_TENSOR.A_ENC_SEANET_CONV_IN: "a.seanet.conv_in",
+ MODEL_TENSOR.A_ENC_SEANET_CONV_OUT: "a.seanet.conv_out",
+ MODEL_TENSOR.A_ENC_SEANET_RES_CONV1: "a.seanet.blk.{bid}.res_conv1",
+ MODEL_TENSOR.A_ENC_SEANET_RES_CONV2: "a.seanet.blk.{bid}.res_conv2",
+ MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV: "a.seanet.blk.{bid}.scale_conv",
+ MODEL_TENSOR.A_ENC_ATTN_SCALE: "a.blk.{bid}.ls1",
+ MODEL_TENSOR.A_ENC_FFN_SCALE_LS: "a.blk.{bid}.ls2",
+ MODEL_TENSOR.A_ENC_SPEAKER_PROJ: "a.speaker_proj",
+ MODEL_TENSOR.A_GEN_FLOW_INPUT_PROJ: "a.gen.flow.input_proj",
+ MODEL_TENSOR.A_GEN_FLOW_COND_EMBD: "a.gen.flow.cond_embd",
+ MODEL_TENSOR.A_GEN_FLOW_TIME_FREQS: "a.gen.flow.time.{bid}.freqs",
+ MODEL_TENSOR.A_GEN_FLOW_TIME_UP: "a.gen.flow.time.{bid}.up",
+ MODEL_TENSOR.A_GEN_FLOW_TIME_DOWN: "a.gen.flow.time.{bid}.down",
+ MODEL_TENSOR.A_GEN_FLOW_TIME_NORM: "a.gen.flow.time.{bid}.norm",
+ MODEL_TENSOR.A_GEN_FLOW_BLK_NORM: "a.gen.flow.blk.{bid}.norm",
+ MODEL_TENSOR.A_GEN_FLOW_BLK_UP: "a.gen.flow.blk.{bid}.up",
+ MODEL_TENSOR.A_GEN_FLOW_BLK_DOWN: "a.gen.flow.blk.{bid}.down",
+ MODEL_TENSOR.A_GEN_FLOW_BLK_ADA: "a.gen.flow.blk.{bid}.ada",
+ MODEL_TENSOR.A_GEN_FLOW_FINAL_ADA: "a.gen.flow.final.ada",
+ MODEL_TENSOR.A_GEN_FLOW_FINAL_PROJ: "a.gen.flow.final.proj",
+ MODEL_TENSOR.A_GEN_OUT_EOS: "a.gen.out_eos",
+ MODEL_TENSOR.A_GEN_INPUT_LINEAR: "a.gen.input_linear",
+ MODEL_TENSOR.A_GEN_EMB_MEAN: "a.gen.emb_mean",
+ MODEL_TENSOR.A_GEN_EMB_STD: "a.gen.emb_std",
+ MODEL_TENSOR.A_GEN_WAV_QUANT_OUT: "a.gen.wav.quant_out",
+ MODEL_TENSOR.A_GEN_WAV_UPSAMPLE: "a.gen.wav.upsample",
+ MODEL_TENSOR.A_GEN_WAV_SEANET_CONV_IN: "a.gen.wav.seanet.conv_in",
+ MODEL_TENSOR.A_GEN_WAV_SEANET_CONV_OUT: "a.gen.wav.seanet.conv_out",
+ MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV1: "a.gen.wav.seanet.blk.{bid}.res_conv1",
+ MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV2: "a.gen.wav.seanet.blk.{bid}.res_conv2",
+ MODEL_TENSOR.A_GEN_WAV_SEANET_SCALE_CONV: "a.gen.wav.seanet.blk.{bid}.scale_conv",
+ MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",
+ MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",
+ MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",
+ MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid",
+ MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n
+ MODEL_TENSOR.A_MM_SOFT_EMB_NORM: "mm.a.soft_emb_norm", # gemma3n
+ MODEL_TENSOR.A_MM_EMBEDDING: "mm.a.embedding", # gemma3n
+ MODEL_TENSOR.A_MM_HARD_EMB_NORM: "mm.a.hard_emb_norm", # gemma3n
+ MODEL_TENSOR.A_MM_CODE_EMBD: "mm.a.code_embd",
+ MODEL_TENSOR.A_MM_LOCAL_ATTN_Q: "mm.a.local_blk.{bid}.attn_q",
+ MODEL_TENSOR.A_MM_LOCAL_ATTN_K: "mm.a.local_blk.{bid}.attn_k",
+ MODEL_TENSOR.A_MM_LOCAL_ATTN_V: "mm.a.local_blk.{bid}.attn_v",
+ MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT: "mm.a.local_blk.{bid}.attn_out",
+ MODEL_TENSOR.A_MM_LOCAL_FFN_GATE: "mm.a.local_blk.{bid}.ffn_gate",
+ MODEL_TENSOR.A_MM_LOCAL_FFN_UP: "mm.a.local_blk.{bid}.ffn_up",
+ MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN: "mm.a.local_blk.{bid}.ffn_down",
+ MODEL_TENSOR.A_MM_LOCAL_LN1: "mm.a.local_blk.{bid}.ln1",
+ MODEL_TENSOR.A_MM_LOCAL_LN2: "mm.a.local_blk.{bid}.ln2",
+ MODEL_TENSOR.A_MM_LOCAL_NORM: "mm.a.local_norm",
+ MODEL_TENSOR.A_PER_DIM_K_SCALE: "a.blk.{bid}.per_dim_k_scale", # gemma4
+ MODEL_TENSOR.A_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale", # gemma4
+ # lfm2 audio
+ MODEL_TENSOR.A_ENC_NORM_CONV: "a.blk.{bid}.norm_conv",
+ MODEL_TENSOR.A_ENC_LINEAR_POS: "a.blk.{bid}.linear_pos",
+ MODEL_TENSOR.A_ENC_POS_BIAS_U: "a.blk.{bid}.pos_bias_u",
+ MODEL_TENSOR.A_ENC_POS_BIAS_V: "a.blk.{bid}.pos_bias_v",
+ MODEL_TENSOR.A_ENC_OUT: "a.pre_encode.out",
+ MODEL_TENSOR.A_ENC_CONV_DW: "a.blk.{bid}.conv_dw",
+ MODEL_TENSOR.A_ENC_CONV_NORM: "a.blk.{bid}.conv_norm",
+ MODEL_TENSOR.A_ENC_CONV_PW1: "a.blk.{bid}.conv_pw1",
+ MODEL_TENSOR.A_ENC_CONV_PW2: "a.blk.{bid}.conv_pw2",
+ MODEL_TENSOR.A_ENC_CONV_NORM_MEAN: "a.blk.{bid}.conv_norm_mean",
+ MODEL_TENSOR.A_ENC_CONV_NORM_VAR: "a.blk.{bid}.conv_norm_var",
+ MODEL_TENSOR.A_ENC_MEL_FILTERS: "a.mel_filters",
+ MODEL_TENSOR.A_ENC_WINDOW: "a.window",
+ MODEL_TENSOR.A_CTC_OUT: "a.enc_ctc_out",
+ MODEL_TENSOR.A_CTC_OUT_MID: "a.enc_ctc_out_mid",
+ MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB: "a.blk.{bid}.attn_rel_pos_emb",
+ # qformer projector
+ MODEL_TENSOR.A_QF_PROJ_QUERY: "a.proj_query",
+ MODEL_TENSOR.A_QF_PROJ_NORM: "a.proj_norm",
+ MODEL_TENSOR.A_QF_PROJ_LINEAR: "a.proj_linear",
+ MODEL_TENSOR.A_QF_SELF_ATTN_Q: "a.proj_blk.{bid}.self_attn_q",
+ MODEL_TENSOR.A_QF_SELF_ATTN_K: "a.proj_blk.{bid}.self_attn_k",
+ MODEL_TENSOR.A_QF_SELF_ATTN_V: "a.proj_blk.{bid}.self_attn_v",
+ MODEL_TENSOR.A_QF_SELF_ATTN_O: "a.proj_blk.{bid}.self_attn_out",
+ MODEL_TENSOR.A_QF_SELF_ATTN_NORM: "a.proj_blk.{bid}.self_attn_norm",
+ MODEL_TENSOR.A_QF_CROSS_ATTN_Q: "a.proj_blk.{bid}.cross_attn_q",
+ MODEL_TENSOR.A_QF_CROSS_ATTN_K: "a.proj_blk.{bid}.cross_attn_k",
+ MODEL_TENSOR.A_QF_CROSS_ATTN_V: "a.proj_blk.{bid}.cross_attn_v",
+ MODEL_TENSOR.A_QF_CROSS_ATTN_O: "a.proj_blk.{bid}.cross_attn_out",
+ MODEL_TENSOR.A_QF_CROSS_ATTN_NORM: "a.proj_blk.{bid}.cross_attn_norm",
+ MODEL_TENSOR.A_QF_FFN_UP: "a.proj_blk.{bid}.ffn_up",
+ MODEL_TENSOR.A_QF_FFN_DOWN: "a.proj_blk.{bid}.ffn_down",
+ MODEL_TENSOR.A_QF_FFN_NORM: "a.proj_blk.{bid}.ffn_norm",
+ # NextN/MTP
+ MODEL_TENSOR.NEXTN_PROJ_PRE: "nextn.pre_projection",
+ MODEL_TENSOR.NEXTN_PROJ_POST: "nextn.post_projection",
+ MODEL_TENSOR.NEXTN_EH_PROJ: "blk.{bid}.nextn.eh_proj",
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS: "blk.{bid}.nextn.embed_tokens",
+ MODEL_TENSOR.NEXTN_ENORM: "blk.{bid}.nextn.enorm",
+ MODEL_TENSOR.NEXTN_HNORM: "blk.{bid}.nextn.hnorm",
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD: "blk.{bid}.nextn.shared_head_head",
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: "blk.{bid}.nextn.shared_head_norm",
+ MODEL_TENSOR.NEXTN_HC_HEAD_NORM: "blk.{bid}.nextn.hc_head_norm",
+ MODEL_TENSOR.NEXTN_HC_HEAD_DOWN: "blk.{bid}.nextn.hc_head_down",
+ MODEL_TENSOR.NEXTN_HC_HEAD_UP: "blk.{bid}.nextn.hc_head_up",
+ MODEL_TENSOR.FC: "fc",
+ MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1",
+ MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2",
+ MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj",
+ MODEL_TENSOR.DFLASH_ATTN_CONV_BASE: "blk.{bid}.attn_conv_base",
+ MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ: "blk.{bid}.attn_conv_proj",
+ MODEL_TENSOR.DFLASH_FFN_CONV_BASE: "blk.{bid}.ffn_conv_base",
+ MODEL_TENSOR.DFLASH_FFN_CONV_PROJ: "blk.{bid}.ffn_conv_proj",
+ MODEL_TENSOR.DFLASH_SELECTOR_PREV: "selector_predecessor",
+ MODEL_TENSOR.DFLASH_SELECTOR_NEXT: "selector_successor",
+ MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN: "selector_hidden",
+ MODEL_TENSOR.D2T: "d2t",
+ MODEL_TENSOR.ATTN_V_GATE: "blk.{bid}.attn_v_gate",
+ MODEL_TENSOR.ATTN_V_EXP: "blk.{bid}.attn_v_exps",
+}
+
+MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
+ MODEL_ARCH.MMPROJ: [
+ MODEL_TENSOR.V_MMPROJ,
+ MODEL_TENSOR.V_MMPROJ_FC,
+ MODEL_TENSOR.V_MMPROJ_MLP,
+ MODEL_TENSOR.V_MMPROJ_PEG,
+ MODEL_TENSOR.V_ENC_EMBD_CLS,
+ MODEL_TENSOR.V_ENC_EMBD_PATCH,
+ MODEL_TENSOR.V_ENC_EMBD_NORM,
+ MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM,
+ MODEL_TENSOR.V_ENC_EMBD_POS,
+ MODEL_TENSOR.V_ENC_EMBD_IMGNL,
+ MODEL_TENSOR.V_ENC_EMBD_VSEP,
+ MODEL_TENSOR.V_TOK_EMBD_SEP,
+ MODEL_TENSOR.V_ENC_INPUT_NORM,
+ MODEL_TENSOR.V_ENC_ATTN_QKV,
+ MODEL_TENSOR.V_ENC_ATTN_Q,
+ MODEL_TENSOR.V_ENC_ATTN_Q_NORM,
+ MODEL_TENSOR.V_ENC_ATTN_K,
+ MODEL_TENSOR.V_ENC_ATTN_K_NORM,
+ MODEL_TENSOR.V_ENC_ATTN_V,
+ MODEL_TENSOR.V_ENC_ATTN_O,
+ MODEL_TENSOR.V_ENC_ATTN_O_NORM,
+ MODEL_TENSOR.V_ENC_ATTN_SINKS,
+ MODEL_TENSOR.V_ENC_POST_ATTN_NORM,
+ MODEL_TENSOR.V_ENC_FFN_UP,
+ MODEL_TENSOR.V_ENC_FFN_GATE,
+ MODEL_TENSOR.V_ENC_FFN_DOWN,
+ MODEL_TENSOR.V_ENC_FFN_GATE_INP,
+ MODEL_TENSOR.V_ENC_FFN_GATE_EXPS,
+ MODEL_TENSOR.V_ENC_FFN_UP_EXPS,
+ MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS,
+ MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B,
+ MODEL_TENSOR.V_ENC_ATTN_POST_NORM,
+ MODEL_TENSOR.V_ENC_FFN_POST_NORM,
+ MODEL_TENSOR.V_LAYER_SCALE_1,
+ MODEL_TENSOR.V_LAYER_SCALE_2,
+ MODEL_TENSOR.V_LAYER_OUT_SCALE,
+ MODEL_TENSOR.V_PRE_NORM,
+ MODEL_TENSOR.V_POST_NORM,
+ MODEL_TENSOR.V_MM_POST_NORM,
+ MODEL_TENSOR.V_MM_INP_PROJ,
+ MODEL_TENSOR.V_MM_INP_NORM,
+ MODEL_TENSOR.V_MM_SOFT_EMB_NORM,
+ MODEL_TENSOR.V_MM_EMBEDDING,
+ MODEL_TENSOR.V_MM_HARD_EMB_NORM,
+ MODEL_TENSOR.V_ENC_CONV_STEM,
+ MODEL_TENSOR.V_ENC_CONV_STEM_NORM,
+ MODEL_TENSOR.V_ENC_MSFA_EXP,
+ MODEL_TENSOR.V_ENC_MSFA_EXP_NORM,
+ MODEL_TENSOR.V_ENC_MSFA_PROJ,
+ MODEL_TENSOR.V_ENC_MSFA_PROJ_NORM,
+ MODEL_TENSOR.V_ENC_MSFA_NORM,
+ MODEL_TENSOR.V_RESMPL_POS_EMBD_K,
+ MODEL_TENSOR.V_RESMPL_ATTN_Q,
+ MODEL_TENSOR.V_RESMPL_ATTN_K,
+ MODEL_TENSOR.V_RESMPL_ATTN_V,
+ MODEL_TENSOR.V_RESMPL_ATTN_OUT,
+ MODEL_TENSOR.V_RESMPL_KV,
+ MODEL_TENSOR.V_RESMPL_KV_NORM,
+ MODEL_TENSOR.V_RESMPL_POST_NORM,
+ MODEL_TENSOR.V_RESMPL_Q_NORM,
+ MODEL_TENSOR.V_RESMPL_PROJ,
+ MODEL_TENSOR.V_RESMPL_QUERY,
+ MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK,
+ MODEL_TENSOR.V_TOK_EMBD_IMG_START,
+ MODEL_TENSOR.V_TOK_EMBD_IMG_END,
+ MODEL_TENSOR.V_TOK_EMBD_IMG_PAD,
+ MODEL_TENSOR.V_MM_PATCH_MERGER,
+ MODEL_TENSOR.V_MM_MERGER_FC1,
+ MODEL_TENSOR.V_MM_MERGER_FC2,
+ MODEL_TENSOR.V_DS_NORM,
+ MODEL_TENSOR.V_DS_FC1,
+ MODEL_TENSOR.V_DS_FC2,
+ MODEL_TENSOR.V_MERGER_LN1,
+ MODEL_TENSOR.V_MERGER_ATTN_Q,
+ MODEL_TENSOR.V_MERGER_ATTN_K,
+ MODEL_TENSOR.V_MERGER_ATTN_V,
+ MODEL_TENSOR.V_MERGER_ATTN_O,
+ MODEL_TENSOR.V_MERGER_DS_LN,
+ MODEL_TENSOR.V_MERGER_DS_UP,
+ MODEL_TENSOR.V_MERGER_DS_DOWN,
+ MODEL_TENSOR.V_MM_POST_FC_NORM,
+ MODEL_TENSOR.V_MM_UP,
+ MODEL_TENSOR.V_MM_DOWN,
+ MODEL_TENSOR.V_MM_GATE,
+ MODEL_TENSOR.V_TOK_BOI,
+ MODEL_TENSOR.V_TOK_EOI,
+ MODEL_TENSOR.V_MM_PRE_NORM,
+ MODEL_TENSOR.V_TOK_IMG_BEGIN,
+ MODEL_TENSOR.V_TOK_IMG_END,
+ MODEL_TENSOR.V_STD_BIAS,
+ MODEL_TENSOR.V_STD_SCALE,
+ MODEL_TENSOR.V_SAM_POS_EMBD,
+ MODEL_TENSOR.V_SAM_PATCH_EMBD,
+ MODEL_TENSOR.V_SAM_PRE_NORM,
+ MODEL_TENSOR.V_SAM_POST_NORM,
+ MODEL_TENSOR.V_SAM_ATTN_POS_H,
+ MODEL_TENSOR.V_SAM_ATTN_POS_W,
+ MODEL_TENSOR.V_SAM_ATTN_QKV,
+ MODEL_TENSOR.V_SAM_ATTN_OUT,
+ MODEL_TENSOR.V_SAM_MLP_LIN_1,
+ MODEL_TENSOR.V_SAM_MLP_LIN_2,
+ MODEL_TENSOR.V_SAM_NECK,
+ MODEL_TENSOR.V_SAM_NET_2,
+ MODEL_TENSOR.V_SAM_NET_3,
+ MODEL_TENSOR.V_RESMPL_QUERY_768,
+ MODEL_TENSOR.V_RESMPL_QUERY_1024,
+ MODEL_TENSOR.V_PROJ_NORM,
+ MODEL_TENSOR.V_QF_PROJ_QUERY,
+ MODEL_TENSOR.V_QF_PROJ_NORM,
+ MODEL_TENSOR.V_QF_PROJ_LINEAR,
+ MODEL_TENSOR.V_QF_SELF_ATTN_Q,
+ MODEL_TENSOR.V_QF_SELF_ATTN_K,
+ MODEL_TENSOR.V_QF_SELF_ATTN_V,
+ MODEL_TENSOR.V_QF_SELF_ATTN_O,
+ MODEL_TENSOR.V_QF_SELF_ATTN_NORM,
+ MODEL_TENSOR.V_QF_CROSS_ATTN_Q,
+ MODEL_TENSOR.V_QF_CROSS_ATTN_K,
+ MODEL_TENSOR.V_QF_CROSS_ATTN_V,
+ MODEL_TENSOR.V_QF_CROSS_ATTN_O,
+ MODEL_TENSOR.V_QF_CROSS_ATTN_NORM,
+ MODEL_TENSOR.V_QF_FFN_UP,
+ MODEL_TENSOR.V_QF_FFN_DOWN,
+ MODEL_TENSOR.V_QF_FFN_NORM,
+ MODEL_TENSOR.V_QF_PROJ_NORM,
+ MODEL_TENSOR.V_MULTI_PROJ_IMG_POS,
+ MODEL_TENSOR.V_MULTI_PROJ_QUERY,
+ MODEL_TENSOR.V_MULTI_PROJ_LINEAR,
+ MODEL_TENSOR.V_MULTI_PROJ_NORM,
+ MODEL_TENSOR.V_MULTI_PROJ_POST_NORM,
+ # audio
+ MODEL_TENSOR.A_ENC_EMBD_POS,
+ MODEL_TENSOR.A_ENC_EMBD_NORM,
+ MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS,
+ MODEL_TENSOR.A_ENC_INP_PROJ,
+ MODEL_TENSOR.A_ENC_CONV1D,
+ MODEL_TENSOR.A_ENC_CONV2D,
+ MODEL_TENSOR.A_ENC_CONV_OUT,
+ MODEL_TENSOR.A_ENC_CONV1D_NORM,
+ MODEL_TENSOR.A_PRE_NORM,
+ MODEL_TENSOR.A_POST_NORM,
+ MODEL_TENSOR.A_ENC_LAYER_PRE_NORM,
+ MODEL_TENSOR.A_ENC_ATTN_Q,
+ MODEL_TENSOR.A_ENC_ATTN_K,
+ MODEL_TENSOR.A_ENC_ATTN_V,
+ MODEL_TENSOR.A_ENC_ATTN_POST_NORM,
+ MODEL_TENSOR.A_ENC_ATTN_PRE_NORM,
+ MODEL_TENSOR.A_ENC_ATTN_K_REL,
+ MODEL_TENSOR.A_ENC_PER_DIM_SCALE,
+ MODEL_TENSOR.A_ENC_INPUT_NORM,
+ MODEL_TENSOR.A_ENC_OUTPUT,
+ MODEL_TENSOR.A_ENC_OUTPUT_NORM,
+ MODEL_TENSOR.A_ENC_FFN_NORM,
+ MODEL_TENSOR.A_ENC_FFN_POST_NORM,
+ MODEL_TENSOR.A_ENC_FFN_SCALE,
+ MODEL_TENSOR.A_ENC_FFN_UP,
+ MODEL_TENSOR.A_ENC_FFN_GATE,
+ MODEL_TENSOR.A_ENC_FFN_DOWN,
+ MODEL_TENSOR.A_ENC_FFN_NORM_1,
+ MODEL_TENSOR.A_ENC_FFN_POST_NORM_1,
+ MODEL_TENSOR.A_ENC_FFN_SCALE_1,
+ MODEL_TENSOR.A_ENC_FFN_UP_1,
+ MODEL_TENSOR.A_ENC_FFN_GATE_1,
+ MODEL_TENSOR.A_ENC_FFN_DOWN_1,
+ MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV,
+ MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM,
+ MODEL_TENSOR.A_ENC_RVQ_CODEBOOK,
+ MODEL_TENSOR.A_MMPROJ,
+ MODEL_TENSOR.A_MMPROJ_FC,
+ MODEL_TENSOR.A_MM_NORM_PRE,
+ MODEL_TENSOR.A_MM_NORM_MID,
+ MODEL_TENSOR.A_MM_CODE_EMBD,
+ MODEL_TENSOR.A_MM_LOCAL_ATTN_Q,
+ MODEL_TENSOR.A_MM_LOCAL_ATTN_K,
+ MODEL_TENSOR.A_MM_LOCAL_ATTN_V,
+ MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT,
+ MODEL_TENSOR.A_MM_LOCAL_FFN_GATE,
+ MODEL_TENSOR.A_MM_LOCAL_FFN_UP,
+ MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN,
+ MODEL_TENSOR.A_MM_LOCAL_LN1,
+ MODEL_TENSOR.A_MM_LOCAL_LN2,
+ MODEL_TENSOR.A_MM_LOCAL_NORM,
+ MODEL_TENSOR.A_ENC_NORM_CONV,
+ MODEL_TENSOR.A_ENC_LINEAR_POS,
+ MODEL_TENSOR.A_ENC_POS_BIAS_U,
+ MODEL_TENSOR.A_ENC_POS_BIAS_V,
+ MODEL_TENSOR.A_ENC_OUT,
+ MODEL_TENSOR.A_ENC_CONV_DW,
+ MODEL_TENSOR.A_ENC_CONV_NORM,
+ MODEL_TENSOR.A_ENC_CONV_PW1,
+ MODEL_TENSOR.A_ENC_CONV_PW2,
+ MODEL_TENSOR.A_ENC_CONV_RES2,
+ MODEL_TENSOR.A_ENC_SE_CONV1,
+ MODEL_TENSOR.A_ENC_SE_CONV2,
+ MODEL_TENSOR.A_ENC_ASP_ATTN,
+ MODEL_TENSOR.A_ENC_ASP_TDNN,
+ MODEL_TENSOR.A_GEN_CODE_PROJ_IN,
+ MODEL_TENSOR.A_GEN_CODE_EMBD,
+ MODEL_TENSOR.A_GEN_CODE_HEAD,
+ MODEL_TENSOR.A_GEN_CODE_OUT_EMBD,
+ MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
+ MODEL_TENSOR.A_GEN_CODE_ATTN_Q,
+ MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM,
+ MODEL_TENSOR.A_GEN_CODE_ATTN_K,
+ MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM,
+ MODEL_TENSOR.A_GEN_CODE_ATTN_V,
+ MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
+ MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
+ MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
+ MODEL_TENSOR.A_GEN_CODE_FFN_UP,
+ MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
+ MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM,
+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_IN,
+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_OUT,
+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_CB,
+ MODEL_TENSOR.A_GEN_WAV_PRE_CONV,
+ MODEL_TENSOR.A_GEN_WAV_TFM_IN_PROJ,
+ MODEL_TENSOR.A_GEN_WAV_TFM_OUT_PROJ,
+ MODEL_TENSOR.A_GEN_WAV_TFM_OUTPUT_NORM,
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM,
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q,
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K,
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V,
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT,
+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE,
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM,
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE,
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE,
+ MODEL_TENSOR.A_GEN_WAV_UP_CONV,
+ MODEL_TENSOR.A_GEN_WAV_UP_DWCONV,
+ MODEL_TENSOR.A_GEN_WAV_UP_NORM,
+ MODEL_TENSOR.A_GEN_WAV_UP_PW1,
+ MODEL_TENSOR.A_GEN_WAV_UP_PW2,
+ MODEL_TENSOR.A_GEN_WAV_UP_GAMMA,
+ MODEL_TENSOR.A_GEN_WAV_DAC_ENTRY,
+ MODEL_TENSOR.A_GEN_WAV_DAC_UP_SNAKE,
+ MODEL_TENSOR.A_GEN_WAV_DAC_UP_CONV,
+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT1,
+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV1,
+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT2,
+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV2,
+ MODEL_TENSOR.A_GEN_WAV_DAC_POST_SNAKE,
+ MODEL_TENSOR.A_GEN_WAV_DAC_POST_CONV,
+ MODEL_TENSOR.A_ENC_SEANET_CONV_IN,
+ MODEL_TENSOR.A_ENC_SEANET_CONV_OUT,
+ MODEL_TENSOR.A_ENC_SEANET_RES_CONV1,
+ MODEL_TENSOR.A_ENC_SEANET_RES_CONV2,
+ MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV,
+ MODEL_TENSOR.A_ENC_ATTN_SCALE,
+ MODEL_TENSOR.A_ENC_FFN_SCALE_LS,
+ MODEL_TENSOR.A_ENC_SPEAKER_PROJ,
+ MODEL_TENSOR.A_GEN_FLOW_INPUT_PROJ,
+ MODEL_TENSOR.A_GEN_FLOW_COND_EMBD,
+ MODEL_TENSOR.A_GEN_FLOW_TIME_FREQS,
+ MODEL_TENSOR.A_GEN_FLOW_TIME_UP,
+ MODEL_TENSOR.A_GEN_FLOW_TIME_DOWN,
+ MODEL_TENSOR.A_GEN_FLOW_TIME_NORM,
+ MODEL_TENSOR.A_GEN_FLOW_BLK_NORM,
+ MODEL_TENSOR.A_GEN_FLOW_BLK_UP,
+ MODEL_TENSOR.A_GEN_FLOW_BLK_DOWN,
+ MODEL_TENSOR.A_GEN_FLOW_BLK_ADA,
+ MODEL_TENSOR.A_GEN_FLOW_FINAL_ADA,
+ MODEL_TENSOR.A_GEN_FLOW_FINAL_PROJ,
+ MODEL_TENSOR.A_GEN_OUT_EOS,
+ MODEL_TENSOR.A_GEN_INPUT_LINEAR,
+ MODEL_TENSOR.A_GEN_EMB_MEAN,
+ MODEL_TENSOR.A_GEN_EMB_STD,
+ MODEL_TENSOR.A_GEN_WAV_QUANT_OUT,
+ MODEL_TENSOR.A_GEN_WAV_UPSAMPLE,
+ MODEL_TENSOR.A_GEN_WAV_SEANET_CONV_IN,
+ MODEL_TENSOR.A_GEN_WAV_SEANET_CONV_OUT,
+ MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV1,
+ MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV2,
+ MODEL_TENSOR.A_GEN_WAV_SEANET_SCALE_CONV,
+ MODEL_TENSOR.A_ENC_CONV_NORM_MEAN,
+ MODEL_TENSOR.A_ENC_CONV_NORM_VAR,
+ MODEL_TENSOR.A_ENC_MEL_FILTERS,
+ MODEL_TENSOR.A_ENC_WINDOW,
+ MODEL_TENSOR.A_MM_INP_PROJ,
+ MODEL_TENSOR.A_MM_SOFT_EMB_NORM,
+ MODEL_TENSOR.A_MM_EMBEDDING,
+ MODEL_TENSOR.A_MM_HARD_EMB_NORM,
+ MODEL_TENSOR.A_PER_DIM_K_SCALE,
+ MODEL_TENSOR.A_PER_DIM_SCALE,
+ MODEL_TENSOR.A_CTC_OUT,
+ MODEL_TENSOR.A_CTC_OUT_MID,
+ MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB,
+ # qformer projector
+ MODEL_TENSOR.A_QF_PROJ_QUERY,
+ MODEL_TENSOR.A_QF_PROJ_NORM,
+ MODEL_TENSOR.A_QF_PROJ_LINEAR,
+ MODEL_TENSOR.A_QF_SELF_ATTN_Q,
+ MODEL_TENSOR.A_QF_SELF_ATTN_K,
+ MODEL_TENSOR.A_QF_SELF_ATTN_V,
+ MODEL_TENSOR.A_QF_SELF_ATTN_O,
+ MODEL_TENSOR.A_QF_SELF_ATTN_NORM,
+ MODEL_TENSOR.A_QF_CROSS_ATTN_Q,
+ MODEL_TENSOR.A_QF_CROSS_ATTN_K,
+ MODEL_TENSOR.A_QF_CROSS_ATTN_V,
+ MODEL_TENSOR.A_QF_CROSS_ATTN_O,
+ MODEL_TENSOR.A_QF_CROSS_ATTN_NORM,
+ MODEL_TENSOR.A_QF_FFN_UP,
+ MODEL_TENSOR.A_QF_FFN_DOWN,
+ MODEL_TENSOR.A_QF_FFN_NORM,
+ ],
+ MODEL_ARCH.LLAMA: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.LLAMA4: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
+ MODEL_ARCH.DECI: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.GROK: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.ATTN_OUT_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ ],
+ MODEL_ARCH.GPTNEOX: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.FALCON: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_NORM_2,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.BAICHUAN: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.STARCODER: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.POS_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.BERT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.TOKEN_TYPES,
+ MODEL_TENSOR.POS_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_OUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ MODEL_TENSOR.CLS,
+ MODEL_TENSOR.CLS_OUT,
+ ],
+ MODEL_ARCH.MODERN_BERT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.TOKEN_TYPES,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.CLS,
+ MODEL_TENSOR.CLS_OUT,
+ MODEL_TENSOR.CLS_NORM,
+ ],
+ MODEL_ARCH.NOMIC_BERT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.TOKEN_TYPES,
+ MODEL_TENSOR.POS_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_OUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ ],
+ MODEL_ARCH.NOMIC_BERT_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.TOKEN_TYPES,
+ MODEL_TENSOR.POS_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_OUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ ],
+ MODEL_ARCH.NEO_BERT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ENC_OUTPUT_NORM,
+ MODEL_TENSOR.CLS,
+ MODEL_TENSOR.CLS_OUT,
+ ],
+ MODEL_ARCH.JINA_BERT_V2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.TOKEN_TYPES,
+ MODEL_TENSOR.ATTN_NORM_2,
+ MODEL_TENSOR.ATTN_OUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ MODEL_TENSOR.CLS,
+ ],
+ MODEL_ARCH.JINA_BERT_V3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.TOKEN_TYPES,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_OUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ ],
+ MODEL_ARCH.EUROBERT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_DOWN,
+ ],
+ MODEL_ARCH.MPT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_ACT,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.POS_EMBD,
+ ],
+ MODEL_ARCH.GPTJ: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.REFACT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.BLOOM: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.STABLELM: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ ],
+ MODEL_ARCH.QWEN: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.QWEN2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.DREAM: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.LLADA: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.QWEN2VL: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.QWEN2MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_INP_SHEXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
+ MODEL_ARCH.QWEN3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.QWEN3MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.QWEN3NEXT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_INP_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_BETA_ALPHA,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.HRM_TEXT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.HRM_Z_L_INIT,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.QWEN3VL: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.QWEN3VLMOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.QWEN35: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.CLS_OUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_ALPHA,
+ MODEL_TENSOR.SSM_OUT,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.CLEF: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_ALPHA,
+ MODEL_TENSOR.SSM_OUT,
+ # decision head
+ MODEL_TENSOR.TOKEN_TYPES,
+ MODEL_TENSOR.DEC_ATTN_NORM,
+ MODEL_TENSOR.DEC_ATTN_Q,
+ MODEL_TENSOR.DEC_ATTN_K,
+ MODEL_TENSOR.DEC_ATTN_V,
+ MODEL_TENSOR.DEC_ATTN_OUT,
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM,
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV,
+ MODEL_TENSOR.DEC_CROSS_ATTN_Q,
+ MODEL_TENSOR.DEC_CROSS_ATTN_K,
+ MODEL_TENSOR.DEC_CROSS_ATTN_V,
+ MODEL_TENSOR.DEC_CROSS_ATTN_OUT,
+ MODEL_TENSOR.DEC_FFN_NORM,
+ MODEL_TENSOR.DEC_FFN_DOWN,
+ MODEL_TENSOR.DEC_FFN_UP,
+ MODEL_TENSOR.DECISION_HIDDEN_NORM,
+ MODEL_TENSOR.DECISION_PROJ_MEMORY,
+ MODEL_TENSOR.DECISION_PROJ_QUESTION,
+ MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION,
+ MODEL_TENSOR.DECISION_PROJ_GLOBAL,
+ MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT,
+ MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL,
+ MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM,
+ MODEL_TENSOR.DECISION_FIELD_NORM,
+ MODEL_TENSOR.DECISION_OPTION_NORM,
+ MODEL_TENSOR.DECISION_SCALES,
+ MODEL_TENSOR.DECISION_SCORER,
+ MODEL_TENSOR.DECISION_SCORER_OUT,
+ ],
+ MODEL_ARCH.VLLM_DECISION2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ # decision head
+ MODEL_TENSOR.DECISION_OPTION_NORM,
+ MODEL_TENSOR.DECISION_QUERY_NORM,
+ MODEL_TENSOR.DECISION_KEY,
+ MODEL_TENSOR.DECISION_QUERY,
+ MODEL_TENSOR.DECISION_OPTION_MLP,
+ MODEL_TENSOR.DECISION_QUERY_MLP,
+ MODEL_TENSOR.DECISION_SCORER_OUT,
+ ],
+ MODEL_ARCH.QWEN35MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_INP_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_ALPHA,
+ MODEL_TENSOR.SSM_OUT,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.QWEN4EXP: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ # no OUTPUT_NORM / ATTN_NORM / ATTN_POST_NORM: hyper-connections replace every layer norm
+ MODEL_TENSOR.HC_HEAD_NORM,
+ MODEL_TENSOR.HC_HEAD_DOWN,
+ MODEL_TENSOR.HC_HEAD_UP,
+ MODEL_TENSOR.HC_ATTN_NORM,
+ MODEL_TENSOR.HC_ATTN_DOWN,
+ MODEL_TENSOR.HC_ATTN_UP,
+ MODEL_TENSOR.HC_ATTN_INJECT,
+ MODEL_TENSOR.HC_FFN_NORM,
+ MODEL_TENSOR.HC_FFN_DOWN,
+ MODEL_TENSOR.HC_FFN_UP,
+ MODEL_TENSOR.HC_FFN_INJECT,
+ # full attention layers: ATTN_Q holds [q|gate] interleaved per head
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.INDEXER_Q_PROJ,
+ MODEL_TENSOR.INDEXER_K_PROJ,
+ MODEL_TENSOR.INDEXER_Q_NORM,
+ MODEL_TENSOR.INDEXER_K_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_ALPHA,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_INP_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,
+ MODEL_TENSOR.PLE_KEY,
+ MODEL_TENSOR.PLE_VALUE,
+ MODEL_TENSOR.PLE_NORM_KEY,
+ MODEL_TENSOR.PLE_NORM_QUERY,
+ MODEL_TENSOR.PLE_NORM_CONV,
+ MODEL_TENSOR.PLE_CONV1D,
+ # MTP block: [fc_embedding | fc_hidden] as eh_proj, its own hyper-connection mixer as the head
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_HC_HEAD_NORM,
+ MODEL_TENSOR.NEXTN_HC_HEAD_DOWN,
+ MODEL_TENSOR.NEXTN_HC_HEAD_UP,
+ ],
+ MODEL_ARCH.PLAMO: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.PLAMO2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_X,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_D,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.SSM_DT_NORM,
+ MODEL_TENSOR.SSM_B_NORM,
+ MODEL_TENSOR.SSM_C_NORM,
+ ],
+ MODEL_ARCH.PLAMO3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_POST_NORM,
+ ],
+ MODEL_ARCH.GPT2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.POS_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.PHI2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.PHI3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FACTORS_LONG,
+ MODEL_TENSOR.ROPE_FACTORS_SHORT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.PHIMOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FACTORS_LONG,
+ MODEL_TENSOR.ROPE_FACTORS_SHORT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.CODESHELL: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.POS_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.ORION: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.INTERNLM2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.MINICPM: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ROPE_FACTORS_LONG,
+ MODEL_TENSOR.ROPE_FACTORS_SHORT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.MINICPM3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FACTORS_LONG,
+ MODEL_TENSOR.ROPE_FACTORS_SHORT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.GEMMA: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_NORM,
+ ],
+ MODEL_ARCH.GEMMA2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ ],
+ MODEL_ARCH.GEMMA3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ ],
+ MODEL_ARCH.GEMMA3N: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ # altup / laurel
+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,
+ MODEL_TENSOR.PER_LAYER_MODEL_PROJ,
+ MODEL_TENSOR.PER_LAYER_INP_GATE,
+ MODEL_TENSOR.PER_LAYER_PROJ,
+ MODEL_TENSOR.PER_LAYER_PROJ_NORM,
+ MODEL_TENSOR.PER_LAYER_POST_NORM,
+ MODEL_TENSOR.ALTUP_PROJ,
+ MODEL_TENSOR.ALTUP_UNEMBD_PROJ,
+ MODEL_TENSOR.ALTUP_CORRECT_COEF,
+ MODEL_TENSOR.ALTUP_CORRECT_SCALE,
+ MODEL_TENSOR.ALTUP_PREDICT_COEF,
+ MODEL_TENSOR.ALTUP_ROUTER,
+ MODEL_TENSOR.ALTUP_ROUTER_NORM,
+ MODEL_TENSOR.LAUREL_L,
+ MODEL_TENSOR.LAUREL_R,
+ MODEL_TENSOR.LAUREL_POST_NORM,
+ ],
+ MODEL_ARCH.GEMMA4: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_PRE_NORM_2,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.FFN_POST_NORM_1,
+ MODEL_TENSOR.FFN_POST_NORM_2,
+ MODEL_TENSOR.LAYER_OUT_SCALE,
+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,
+ MODEL_TENSOR.PER_LAYER_MODEL_PROJ,
+ MODEL_TENSOR.PER_LAYER_INP_GATE,
+ MODEL_TENSOR.PER_LAYER_PROJ,
+ MODEL_TENSOR.PER_LAYER_PROJ_NORM,
+ MODEL_TENSOR.PER_LAYER_POST_NORM,
+ ],
+ MODEL_ARCH.GEMMA4_ASSISTANT: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.MASKED_EMBD_CENTROIDS,
+ MODEL_TENSOR.MASKED_EMBD_ORDERING,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.NEXTN_PROJ_PRE,
+ MODEL_TENSOR.NEXTN_PROJ_POST,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.LAYER_OUT_SCALE,
+ ],
+ MODEL_ARCH.GEMMA_EMBEDDING: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.DENSE_2_OUT,
+ MODEL_TENSOR.DENSE_3_OUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ ],
+ MODEL_ARCH.GEMMA_EMBEDDING2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.LAYER_OUT_SCALE,
+ MODEL_TENSOR.PER_LAYER_MODEL_PROJ,
+ MODEL_TENSOR.PER_LAYER_INP_GATE,
+ MODEL_TENSOR.PER_LAYER_PROJ,
+ MODEL_TENSOR.PER_LAYER_PROJ_NORM,
+ MODEL_TENSOR.PER_LAYER_POST_NORM,
+ ],
+ MODEL_ARCH.STARCODER2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.RWKV6: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_NORM_2,
+ MODEL_TENSOR.TIME_MIX_W1,
+ MODEL_TENSOR.TIME_MIX_W2,
+ MODEL_TENSOR.TIME_MIX_LERP_X,
+ MODEL_TENSOR.TIME_MIX_LERP_K,
+ MODEL_TENSOR.TIME_MIX_LERP_V,
+ MODEL_TENSOR.TIME_MIX_LERP_R,
+ MODEL_TENSOR.TIME_MIX_LERP_G,
+ MODEL_TENSOR.TIME_MIX_LERP_W,
+ MODEL_TENSOR.TIME_MIX_LERP_FUSED,
+ MODEL_TENSOR.TIME_MIX_FIRST,
+ MODEL_TENSOR.TIME_MIX_DECAY,
+ MODEL_TENSOR.TIME_MIX_DECAY_W1,
+ MODEL_TENSOR.TIME_MIX_DECAY_W2,
+ MODEL_TENSOR.TIME_MIX_KEY,
+ MODEL_TENSOR.TIME_MIX_VALUE,
+ MODEL_TENSOR.TIME_MIX_RECEPTANCE,
+ MODEL_TENSOR.TIME_MIX_GATE,
+ MODEL_TENSOR.TIME_MIX_LN,
+ MODEL_TENSOR.TIME_MIX_OUTPUT,
+ MODEL_TENSOR.CHANNEL_MIX_LERP_K,
+ MODEL_TENSOR.CHANNEL_MIX_LERP_R,
+ MODEL_TENSOR.CHANNEL_MIX_KEY,
+ MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE,
+ MODEL_TENSOR.CHANNEL_MIX_VALUE,
+ ],
+ MODEL_ARCH.RWKV6QWEN2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.TIME_MIX_W1,
+ MODEL_TENSOR.TIME_MIX_W2,
+ MODEL_TENSOR.TIME_MIX_LERP_X,
+ MODEL_TENSOR.TIME_MIX_LERP_K,
+ MODEL_TENSOR.TIME_MIX_LERP_V,
+ MODEL_TENSOR.TIME_MIX_LERP_R,
+ MODEL_TENSOR.TIME_MIX_LERP_G,
+ MODEL_TENSOR.TIME_MIX_LERP_W,
+ MODEL_TENSOR.TIME_MIX_LERP_FUSED,
+ MODEL_TENSOR.TIME_MIX_FIRST,
+ MODEL_TENSOR.TIME_MIX_DECAY,
+ MODEL_TENSOR.TIME_MIX_DECAY_W1,
+ MODEL_TENSOR.TIME_MIX_DECAY_W2,
+ MODEL_TENSOR.TIME_MIX_KEY,
+ MODEL_TENSOR.TIME_MIX_VALUE,
+ MODEL_TENSOR.TIME_MIX_RECEPTANCE,
+ MODEL_TENSOR.TIME_MIX_GATE,
+ MODEL_TENSOR.TIME_MIX_LN,
+ MODEL_TENSOR.TIME_MIX_OUTPUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.RWKV7: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_NORM_2,
+ MODEL_TENSOR.TIME_MIX_LERP_FUSED,
+ MODEL_TENSOR.TIME_MIX_W0,
+ MODEL_TENSOR.TIME_MIX_W1,
+ MODEL_TENSOR.TIME_MIX_W2,
+ MODEL_TENSOR.TIME_MIX_A0,
+ MODEL_TENSOR.TIME_MIX_A1,
+ MODEL_TENSOR.TIME_MIX_A2,
+ MODEL_TENSOR.TIME_MIX_V0,
+ MODEL_TENSOR.TIME_MIX_V1,
+ MODEL_TENSOR.TIME_MIX_V2,
+ MODEL_TENSOR.TIME_MIX_G1,
+ MODEL_TENSOR.TIME_MIX_G2,
+ MODEL_TENSOR.TIME_MIX_K_K,
+ MODEL_TENSOR.TIME_MIX_K_A,
+ MODEL_TENSOR.TIME_MIX_R_K,
+ MODEL_TENSOR.TIME_MIX_KEY,
+ MODEL_TENSOR.TIME_MIX_VALUE,
+ MODEL_TENSOR.TIME_MIX_RECEPTANCE,
+ MODEL_TENSOR.TIME_MIX_LN,
+ MODEL_TENSOR.TIME_MIX_OUTPUT,
+ MODEL_TENSOR.CHANNEL_MIX_LERP_K,
+ MODEL_TENSOR.CHANNEL_MIX_KEY,
+ MODEL_TENSOR.CHANNEL_MIX_VALUE,
+ ],
+ MODEL_ARCH.ARWKV7: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.TIME_MIX_LERP_FUSED,
+ MODEL_TENSOR.TIME_MIX_W0,
+ MODEL_TENSOR.TIME_MIX_W1,
+ MODEL_TENSOR.TIME_MIX_W2,
+ MODEL_TENSOR.TIME_MIX_A0,
+ MODEL_TENSOR.TIME_MIX_A1,
+ MODEL_TENSOR.TIME_MIX_A2,
+ MODEL_TENSOR.TIME_MIX_V0,
+ MODEL_TENSOR.TIME_MIX_V1,
+ MODEL_TENSOR.TIME_MIX_V2,
+ MODEL_TENSOR.TIME_MIX_G1,
+ MODEL_TENSOR.TIME_MIX_G2,
+ MODEL_TENSOR.TIME_MIX_K_K,
+ MODEL_TENSOR.TIME_MIX_K_A,
+ MODEL_TENSOR.TIME_MIX_R_K,
+ MODEL_TENSOR.TIME_MIX_KEY,
+ MODEL_TENSOR.TIME_MIX_VALUE,
+ MODEL_TENSOR.TIME_MIX_RECEPTANCE,
+ MODEL_TENSOR.TIME_MIX_LN,
+ MODEL_TENSOR.TIME_MIX_OUTPUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.MAMBA: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_X,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_D,
+ MODEL_TENSOR.SSM_OUT,
+ ],
+ MODEL_ARCH.MAMBA2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_D,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_OUT,
+ ],
+ MODEL_ARCH.MAPLE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.JAMBA: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_X,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_DT_NORM,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_B_NORM,
+ MODEL_TENSOR.SSM_C_NORM,
+ MODEL_TENSOR.SSM_D,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.XVERSE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.COMMAND_R: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ ],
+ MODEL_ARCH.COHERE2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.COHERE2MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.DBRX: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_OUT_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.OLMO: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.OLMO2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.SEED_OSS: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ ],
+ MODEL_ARCH.OLMOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ ],
+ MODEL_ARCH.MUSE_GLIMMER: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ ],
+ MODEL_ARCH.OPENELM: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.ARCTIC: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_NORM_EXP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.DEEPSEEK: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
+ MODEL_ARCH.DEEPSEEK2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ # NextN/MTP tensors
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.DEEPSEEK2OCR: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ ],
+ MODEL_ARCH.DEEPSEEK32: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.INDEXER_K_NORM,
+ MODEL_TENSOR.INDEXER_PROJ,
+ MODEL_TENSOR.INDEXER_ATTN_K,
+ MODEL_TENSOR.INDEXER_ATTN_Q_B,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.DEEPSEEK4: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.HC_HEAD_FN,
+ MODEL_TENSOR.HC_HEAD_BASE,
+ MODEL_TENSOR.HC_HEAD_SCALE,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_SINKS,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV,
+ MODEL_TENSOR.ATTN_KV_NORM,
+ MODEL_TENSOR.ATTN_OUT_A,
+ MODEL_TENSOR.ATTN_OUT_B,
+ MODEL_TENSOR.HC_ATTN_FN,
+ MODEL_TENSOR.HC_ATTN_BASE,
+ MODEL_TENSOR.HC_ATTN_SCALE,
+ MODEL_TENSOR.HC_FFN_FN,
+ MODEL_TENSOR.HC_FFN_BASE,
+ MODEL_TENSOR.HC_FFN_SCALE,
+ MODEL_TENSOR.ATTN_COMPRESSOR_WKV,
+ MODEL_TENSOR.ATTN_COMPRESSOR_WGATE,
+ MODEL_TENSOR.ATTN_COMPRESSOR_APE,
+ MODEL_TENSOR.ATTN_COMPRESSOR_NORM,
+ MODEL_TENSOR.INDEXER_PROJ,
+ MODEL_TENSOR.INDEXER_ATTN_Q_B,
+ MODEL_TENSOR.INDEXER_COMPRESSOR_WKV,
+ MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE,
+ MODEL_TENSOR.INDEXER_COMPRESSOR_APE,
+ MODEL_TENSOR.INDEXER_COMPRESSOR_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_TID2EID,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_EXP_PROBS_B_VL,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.ERNIE4_5_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ ],
+ MODEL_ARCH.PLM: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_DOWN,
+ ],
+ MODEL_ARCH.CHATGLM : [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.GLM4 : [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.GLM4_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ # NextN/MTP tensors
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.GLM_DSA: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.INDEXER_K_NORM,
+ MODEL_TENSOR.INDEXER_PROJ,
+ MODEL_TENSOR.INDEXER_ATTN_K,
+ MODEL_TENSOR.INDEXER_ATTN_Q_B,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.GLM5_NEXT: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ # mHC
+ MODEL_TENSOR.HC_ATTN_FN,
+ MODEL_TENSOR.HC_ATTN_BASE,
+ MODEL_TENSOR.HC_ATTN_SCALE,
+ MODEL_TENSOR.HC_FFN_FN,
+ MODEL_TENSOR.HC_FFN_BASE,
+ MODEL_TENSOR.HC_FFN_SCALE,
+ # KDA (linear attention) layers
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.SSM_CONV1D_Q,
+ MODEL_TENSOR.SSM_CONV1D_K,
+ MODEL_TENSOR.SSM_CONV1D_V,
+ MODEL_TENSOR.SSM_F_A,
+ MODEL_TENSOR.SSM_F_B,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_G_A,
+ MODEL_TENSOR.SSM_G_B,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ # MLA (nope) + DSA layers
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.INDEXER_K_NORM,
+ MODEL_TENSOR.INDEXER_PROJ,
+ MODEL_TENSOR.INDEXER_ATTN_K,
+ MODEL_TENSOR.INDEXER_ATTN_Q_B,
+ MODEL_TENSOR.INDEXER_KPOOL_GATE,
+ MODEL_TENSOR.INDEXER_KPOOL_APE,
+ # FFN
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.BITNET: [
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.ATTN_SUB_NORM,
+ MODEL_TENSOR.FFN_SUB_NORM,
+ ],
+ MODEL_ARCH.T5: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.DEC_ATTN_NORM,
+ MODEL_TENSOR.DEC_ATTN_Q,
+ MODEL_TENSOR.DEC_ATTN_K,
+ MODEL_TENSOR.DEC_ATTN_V,
+ MODEL_TENSOR.DEC_ATTN_OUT,
+ MODEL_TENSOR.DEC_ATTN_REL_B,
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM,
+ MODEL_TENSOR.DEC_CROSS_ATTN_Q,
+ MODEL_TENSOR.DEC_CROSS_ATTN_K,
+ MODEL_TENSOR.DEC_CROSS_ATTN_V,
+ MODEL_TENSOR.DEC_CROSS_ATTN_OUT,
+ MODEL_TENSOR.DEC_CROSS_ATTN_REL_B,
+ MODEL_TENSOR.DEC_FFN_NORM,
+ MODEL_TENSOR.DEC_FFN_GATE,
+ MODEL_TENSOR.DEC_FFN_DOWN,
+ MODEL_TENSOR.DEC_FFN_UP,
+ MODEL_TENSOR.DEC_OUTPUT_NORM,
+ MODEL_TENSOR.ENC_ATTN_NORM,
+ MODEL_TENSOR.ENC_ATTN_Q,
+ MODEL_TENSOR.ENC_ATTN_K,
+ MODEL_TENSOR.ENC_ATTN_V,
+ MODEL_TENSOR.ENC_ATTN_OUT,
+ MODEL_TENSOR.ENC_ATTN_REL_B,
+ MODEL_TENSOR.ENC_FFN_NORM,
+ MODEL_TENSOR.ENC_FFN_GATE,
+ MODEL_TENSOR.ENC_FFN_DOWN,
+ MODEL_TENSOR.ENC_FFN_UP,
+ MODEL_TENSOR.ENC_OUTPUT_NORM,
+ ],
+ MODEL_ARCH.T5ENCODER: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ENC_ATTN_NORM,
+ MODEL_TENSOR.ENC_ATTN_Q,
+ MODEL_TENSOR.ENC_ATTN_K,
+ MODEL_TENSOR.ENC_ATTN_V,
+ MODEL_TENSOR.ENC_ATTN_OUT,
+ MODEL_TENSOR.ENC_ATTN_REL_B,
+ MODEL_TENSOR.ENC_FFN_NORM,
+ MODEL_TENSOR.ENC_FFN_GATE,
+ MODEL_TENSOR.ENC_FFN_DOWN,
+ MODEL_TENSOR.ENC_FFN_UP,
+ MODEL_TENSOR.ENC_OUTPUT_NORM,
+ ],
+ MODEL_ARCH.JAIS: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.JAIS2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.NEMOTRON: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.NEMOTRON_H: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_D,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.NEMOTRON_H_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_D,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ # experts
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ # expert latent
+ MODEL_TENSOR.MOE_LATENT_DOWN,
+ MODEL_TENSOR.MOE_LATENT_UP,
+ # shared expert
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ # NextN/MTP (draft head)
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.EXAONE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.EXAONE4: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_POST_NORM,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.EXAONE_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.GRANITE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.GRANITE_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ ],
+ MODEL_ARCH.GRANITE_HYBRID: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_D,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ # MoE
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ # Dense
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.GRANITE_SWITCH: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.GRANITE_SWA: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_SINKS,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ # MoE (GraniteMoeSWA)
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ # Shared expert - gate+up kept fused in FFN_UP_SHEXP (LLM_FFN_SWIGLU)
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ ],
+ MODEL_ARCH.CHAMELEON: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.WAVTOKENIZER_DEC: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.CONV1D,
+ MODEL_TENSOR.CONVNEXT_DW,
+ MODEL_TENSOR.CONVNEXT_NORM,
+ MODEL_TENSOR.CONVNEXT_PW1,
+ MODEL_TENSOR.CONVNEXT_PW2,
+ MODEL_TENSOR.CONVNEXT_GAMMA,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.POSNET_CONV1,
+ MODEL_TENSOR.POSNET_CONV2,
+ MODEL_TENSOR.POSNET_NORM,
+ MODEL_TENSOR.POSNET_NORM1,
+ MODEL_TENSOR.POSNET_NORM2,
+ MODEL_TENSOR.POSNET_ATTN_NORM,
+ MODEL_TENSOR.POSNET_ATTN_Q,
+ MODEL_TENSOR.POSNET_ATTN_K,
+ MODEL_TENSOR.POSNET_ATTN_V,
+ MODEL_TENSOR.POSNET_ATTN_OUT,
+ ],
+ MODEL_ARCH.BAILINGMOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
+ MODEL_ARCH.BAILINGMOE2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ ],
+ MODEL_ARCH.BAILINGMOE3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.SSM_CONV1D_Q,
+ MODEL_TENSOR.SSM_CONV1D_K,
+ MODEL_TENSOR.SSM_CONV1D_V,
+ MODEL_TENSOR.SSM_F_A,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_G_A,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ ],
+ MODEL_ARCH.DOTS1: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
+ MODEL_ARCH.DOTS3NOTE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.INDEXER_K_NORM,
+ MODEL_TENSOR.INDEXER_PROJ,
+ MODEL_TENSOR.INDEXER_ATTN_K,
+ MODEL_TENSOR.INDEXER_ATTN_Q_B,
+ # NextN/MTP tensors - preserved but unused
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.ARCEE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.AFMOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_PRE_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ ],
+ MODEL_ARCH.LAGUNA: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ ],
+ MODEL_ARCH.ERNIE4_5: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.PADDLEOCR: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.FALCON_H1: [
+ # Token embedding
+ MODEL_TENSOR.TOKEN_EMBD,
+
+ # Input layernorm
+ MODEL_TENSOR.ATTN_NORM,
+
+ # Attention components
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q, # Query projection
+ MODEL_TENSOR.ATTN_K, # Key projection
+ MODEL_TENSOR.ATTN_V, # Value projection
+ MODEL_TENSOR.ATTN_OUT, # Output projection
+
+ # SSM components (Mamba2 specific)
+ MODEL_TENSOR.SSM_IN, # Input projection for SSM
+ MODEL_TENSOR.SSM_CONV1D, # Convolution layer
+ MODEL_TENSOR.SSM_DT, # Delta time projection
+ MODEL_TENSOR.SSM_A, # A parameter (log form)
+ MODEL_TENSOR.SSM_D, # D parameter
+ MODEL_TENSOR.SSM_NORM, # Normalization in SSM
+ MODEL_TENSOR.SSM_OUT, # Output projection
+
+ # Pre-feedforward layernorm
+ MODEL_TENSOR.FFN_PRE_NORM,
+
+ # Feed-forward network components
+ MODEL_TENSOR.FFN_GATE, # Gate projection (SwiGLU)
+ MODEL_TENSOR.FFN_DOWN, # Down projection
+ MODEL_TENSOR.FFN_UP, # Up projection
+
+ # Post-feedforward layernorm
+ MODEL_TENSOR.OUTPUT_NORM, # Final layer norm
+ MODEL_TENSOR.OUTPUT, # Output projection (lm_head)
+ ],
+ MODEL_ARCH.HUNYUAN_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
+ MODEL_ARCH.HUNYUAN_DENSE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.HUNYUAN_VL: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.HY_V3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ # NextN/MTP tensors (draft head)
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.HY_V4: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.HC_HEAD_FN,
+ MODEL_TENSOR.HC_HEAD_BASE,
+ MODEL_TENSOR.HC_HEAD_SCALE,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_SINKS,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.INDEXER_K_NORM,
+ MODEL_TENSOR.INDEXER_PROJ,
+ MODEL_TENSOR.INDEXER_ATTN_K,
+ MODEL_TENSOR.INDEXER_ATTN_Q_B,
+ MODEL_TENSOR.HC_ATTN_FN,
+ MODEL_TENSOR.HC_ATTN_BASE,
+ MODEL_TENSOR.HC_ATTN_SCALE,
+ MODEL_TENSOR.HC_FFN_FN,
+ MODEL_TENSOR.HC_FFN_BASE,
+ MODEL_TENSOR.HC_FFN_SCALE,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
+ MODEL_ARCH.SMOLLM3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.GPT_OSS: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_SINKS,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.LFM2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.SHORTCONV_CONV,
+ MODEL_TENSOR.SHORTCONV_INPROJ,
+ MODEL_TENSOR.SHORTCONV_OUTPROJ,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.ATTN_NORM, # operator_norm
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.DENSE_2_OUT, # LFM2-ColBert-350M
+ MODEL_TENSOR.TOKEN_TYPES, # decision head
+ MODEL_TENSOR.CLS,
+ MODEL_TENSOR.CLS_NORM,
+ MODEL_TENSOR.CLS_OUT,
+ ],
+ MODEL_ARCH.LFM2MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.SHORTCONV_CONV,
+ MODEL_TENSOR.SHORTCONV_INPROJ,
+ MODEL_TENSOR.SHORTCONV_OUTPROJ,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.ATTN_NORM, # operator_norm
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ ],
+ MODEL_ARCH.SMALLTHINKER: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.APERTUS: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.LLADA_MOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ ],
+ MODEL_ARCH.GROVEMOE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_CHEXP,
+ MODEL_TENSOR.FFN_DOWN_CHEXP,
+ MODEL_TENSOR.FFN_UP_CHEXP,
+ ],
+ MODEL_ARCH.MINIMAX01: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_NORM_2,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.MINIMAXM2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ ],
+ MODEL_ARCH.MINIMAXM3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.INDEXER_Q_PROJ,
+ MODEL_TENSOR.INDEXER_K_PROJ,
+ MODEL_TENSOR.INDEXER_Q_NORM,
+ MODEL_TENSOR.INDEXER_K_NORM,
+ ],
+ MODEL_ARCH.COGVLM: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.VISEXP_ATTN_QKV,
+ MODEL_TENSOR.VISEXP_ATTN_OUT,
+ MODEL_TENSOR.VISEXP_GATE,
+ MODEL_TENSOR.VISEXP_UP,
+ MODEL_TENSOR.VISEXP_DOWN,
+ ],
+ MODEL_ARCH.RND1: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.PANGU_EMBED: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.MISTRAL3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.EAGLE3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_NORM_2,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FC,
+ MODEL_TENSOR.ENC_OUTPUT_NORM,
+ MODEL_TENSOR.D2T,
+ ],
+ MODEL_ARCH.DFLASH: [
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.LAYER_OUT_SCALE,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_SINKS,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV,
+ MODEL_TENSOR.ATTN_KV_NORM,
+ MODEL_TENSOR.ATTN_OUT_A,
+ MODEL_TENSOR.ATTN_OUT_B,
+ MODEL_TENSOR.HC_ATTN_FN,
+ MODEL_TENSOR.HC_ATTN_BASE,
+ MODEL_TENSOR.HC_ATTN_SCALE,
+ MODEL_TENSOR.HC_FFN_FN,
+ MODEL_TENSOR.HC_FFN_BASE,
+ MODEL_TENSOR.HC_FFN_SCALE,
+ MODEL_TENSOR.HC_HEAD_FN,
+ MODEL_TENSOR.HC_HEAD_BASE,
+ MODEL_TENSOR.HC_HEAD_SCALE,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FC,
+ MODEL_TENSOR.ENC_OUTPUT_NORM,
+ MODEL_TENSOR.D2T,
+ # optional DSpark heads
+ MODEL_TENSOR.DSPARK_MARKOV_W1,
+ MODEL_TENSOR.DSPARK_MARKOV_W2,
+ MODEL_TENSOR.DSPARK_CONF_PROJ,
+ MODEL_TENSOR.DFLASH_ATTN_CONV_BASE,
+ MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ,
+ MODEL_TENSOR.DFLASH_FFN_CONV_BASE,
+ MODEL_TENSOR.DFLASH_FFN_CONV_PROJ,
+ MODEL_TENSOR.DFLASH_SELECTOR_PREV,
+ MODEL_TENSOR.DFLASH_SELECTOR_NEXT,
+ MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN,
+ ],
+ MODEL_ARCH.MISTRAL4: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ ],
+ MODEL_ARCH.MIMO2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_SINKS,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.LAYER_OUT_NORM,
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.STEP35: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ # NextN/MTP tensors (Step3p5 draft head)
+ MODEL_TENSOR.NEXTN_EH_PROJ,
+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,
+ MODEL_TENSOR.NEXTN_ENORM,
+ MODEL_TENSOR.NEXTN_HNORM,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
+ ],
+ MODEL_ARCH.SPARK2_5: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.LLAMA_EMBED: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.MAINCODER: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.KIMI_LINEAR: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.SSM_CONV1D_Q,
+ MODEL_TENSOR.SSM_CONV1D_K,
+ MODEL_TENSOR.SSM_CONV1D_V,
+ MODEL_TENSOR.SSM_F_A,
+ MODEL_TENSOR.SSM_F_B,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_G_A,
+ MODEL_TENSOR.SSM_G_B,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
+ MODEL_ARCH.KIMI_K3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_RES_SCORE,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_RES_SCORE,
+ MODEL_TENSOR.FFN_RES_SCORE,
+ # MLA (full-attention layers)
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ # KDA (linear-attention layers)
+ MODEL_TENSOR.SSM_CONV1D_Q,
+ MODEL_TENSOR.SSM_CONV1D_K,
+ MODEL_TENSOR.SSM_CONV1D_V,
+ MODEL_TENSOR.SSM_F_A,
+ MODEL_TENSOR.SSM_F_B,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_G,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ # FFN
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_ROUTED_DOWN,
+ MODEL_TENSOR.FFN_ROUTED_UP,
+ MODEL_TENSOR.FFN_ROUTED_NORM,
+ ],
+ MODEL_ARCH.TALKIE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.LAYER_OUT_SCALE,
+ ],
+ MODEL_ARCH.MELLUM: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ ],
+ MODEL_ARCH.NANBEIGE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.QWEN3TTS: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.POCKETTTS: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
+ MODEL_ARCH.K2HORIZON: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_V_GATE,
+ MODEL_TENSOR.ATTN_V_EXP,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ ],
+}
+
+# tensors that will not be serialized
+MODEL_TENSOR_SKIP: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
+ MODEL_ARCH.LLAMA: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.DECI: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.BAICHUAN: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.QWEN: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.CODESHELL: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.ORION: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.STARCODER2: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.XVERSE: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.DEEPSEEK: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.DEEPSEEK2: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.DEEPSEEK2OCR: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.DEEPSEEK32: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.HY_V4: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.CHATGLM: [
+ MODEL_TENSOR.ROPE_FREQS,
+ ],
+ MODEL_ARCH.NEMOTRON: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.BAILINGMOE: [
+ MODEL_TENSOR.ROPE_FREQS,
+ ],
+ MODEL_ARCH.PANGU_EMBED: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+ MODEL_ARCH.NANBEIGE: [
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.ATTN_ROT_EMBD,
+ ],
+}
+
+#
+# types
+#
+
+
+class TokenType(IntEnum):
+ NORMAL = 1
+ UNKNOWN = 2
+ CONTROL = 3
+ USER_DEFINED = 4
+ UNUSED = 5
+ BYTE = 6
+
+
+class RopeScalingType(Enum):
+ NONE = 'none'
+ LINEAR = 'linear'
+ YARN = 'yarn'
+ LONGROPE = 'longrope'
+
+
+# M-RoPE: input position slot (t, y, x, z) that feeds each RoPE section, in section order
+class RopeSectionOrder(Enum):
+ TYXZ = 'tyxz' # default
+ ZYXT = 'zyxt'
+
+
+class PoolingType(IntEnum):
+ NONE = 0
+ MEAN = 1
+ CLS = 2
+ LAST = 3
+ RANK = 4
+
+
+class GGMLQuantizationType(IntEnum):
+ F32 = 0
+ F16 = 1
+ Q4_0 = 2
+ Q4_1 = 3
+ Q5_0 = 6
+ Q5_1 = 7
+ Q8_0 = 8
+ Q8_1 = 9
+ Q2_K = 10
+ Q3_K = 11
+ Q4_K = 12
+ Q5_K = 13
+ Q6_K = 14
+ Q8_K = 15
+ IQ2_XXS = 16
+ IQ2_XS = 17
+ IQ3_XXS = 18
+ IQ1_S = 19
+ IQ4_NL = 20
+ IQ3_S = 21
+ IQ2_S = 22
+ IQ4_XS = 23
+ I8 = 24
+ I16 = 25
+ I32 = 26
+ I64 = 27
+ F64 = 28
+ IQ1_M = 29
+ BF16 = 30
+ TQ1_0 = 34
+ TQ2_0 = 35
+ MXFP4 = 39
+ NVFP4 = 40
+ Q1_0 = 41
+ Q2_0 = 42
+
+
+class ExpertGatingFuncType(IntEnum):
+ SOFTMAX = 1
+ SIGMOID = 2
+ SQRTSOFTPLUS = 4
+
+
+# TODO: add GGMLFileType from ggml_ftype in ggml.h
+
+
+# from llama_ftype in llama.h
+# ALL VALUES SHOULD BE THE SAME HERE AS THEY ARE OVER THERE.
+class LlamaFileType(IntEnum):
+ ALL_F32 = 0
+ MOSTLY_F16 = 1 # except 1d tensors
+ MOSTLY_Q4_0 = 2 # except 1d tensors
+ MOSTLY_Q4_1 = 3 # except 1d tensors
+ # MOSTLY_Q4_1_SOME_F16 = 4 # tok_embeddings.weight and output.weight are F16
+ # MOSTLY_Q4_2 = 5 # support has been removed
+ # MOSTLY_Q4_3 = 6 # support has been removed
+ MOSTLY_Q8_0 = 7 # except 1d tensors
+ MOSTLY_Q5_0 = 8 # except 1d tensors
+ MOSTLY_Q5_1 = 9 # except 1d tensors
+ MOSTLY_Q2_K = 10 # except 1d tensors
+ MOSTLY_Q3_K_S = 11 # except 1d tensors
+ MOSTLY_Q3_K_M = 12 # except 1d tensors
+ MOSTLY_Q3_K_L = 13 # except 1d tensors
+ MOSTLY_Q4_K_S = 14 # except 1d tensors
+ MOSTLY_Q4_K_M = 15 # except 1d tensors
+ MOSTLY_Q5_K_S = 16 # except 1d tensors
+ MOSTLY_Q5_K_M = 17 # except 1d tensors
+ MOSTLY_Q6_K = 18 # except 1d tensors
+ MOSTLY_IQ2_XXS = 19 # except 1d tensors
+ MOSTLY_IQ2_XS = 20 # except 1d tensors
+ MOSTLY_Q2_K_S = 21 # except 1d tensors
+ MOSTLY_IQ3_XS = 22 # except 1d tensors
+ MOSTLY_IQ3_XXS = 23 # except 1d tensors
+ MOSTLY_IQ1_S = 24 # except 1d tensors
+ MOSTLY_IQ4_NL = 25 # except 1d tensors
+ MOSTLY_IQ3_S = 26 # except 1d tensors
+ MOSTLY_IQ3_M = 27 # except 1d tensors
+ MOSTLY_IQ2_S = 28 # except 1d tensors
+ MOSTLY_IQ2_M = 29 # except 1d tensors
+ MOSTLY_IQ4_XS = 30 # except 1d tensors
+ MOSTLY_IQ1_M = 31 # except 1d tensors
+ MOSTLY_BF16 = 32 # except 1d tensors
+ # MOSTLY_Q4_0_4_4 = 33 # removed from gguf files, use Q4_0 and runtime repack
+ # MOSTLY_Q4_0_4_8 = 34 # removed from gguf files, use Q4_0 and runtime repack
+ # MOSTLY_Q4_0_8_8 = 35 # removed from gguf files, use Q4_0 and runtime repack
+ MOSTLY_TQ1_0 = 36 # except 1d tensors
+ MOSTLY_TQ2_0 = 37 # except 1d tensors
+ MOSTLY_MXFP4_MOE = 38 # except 1d tensors
+ MOSTLY_NVFP4 = 39 # except 1d tensors
+ MOSTLY_Q1_0 = 40 # except 1d tensors
+ MOSTLY_Q2_0 = 41 # except 1d tensors
+
+ GUESSED = 1024 # not specified in the model file
+
+
+class GGUFEndian(IntEnum):
+ LITTLE = 0
+ BIG = 1
+
+
+class GGUFValueType(IntEnum):
+ UINT8 = 0
+ INT8 = 1
+ UINT16 = 2
+ INT16 = 3
+ UINT32 = 4
+ INT32 = 5
+ FLOAT32 = 6
+ BOOL = 7
+ STRING = 8
+ ARRAY = 9
+ UINT64 = 10
+ INT64 = 11
+ FLOAT64 = 12
+
+ @staticmethod
+ def get_type(val: Any) -> GGUFValueType:
+ if isinstance(val, (str, bytes, bytearray)):
+ return GGUFValueType.STRING
+ elif isinstance(val, list):
+ return GGUFValueType.ARRAY
+ elif isinstance(val, float):
+ return GGUFValueType.FLOAT32
+ elif isinstance(val, bool):
+ return GGUFValueType.BOOL
+ elif isinstance(val, int):
+ return GGUFValueType.INT32
+ # TODO: need help with 64-bit types in Python
+ else:
+ raise ValueError(f"Unknown type: {type(val)}")
+
+
+class DecisionType:
+ LAYA = "laya" # head blocks + scorer on the hidden state of one marker token per option
+ OPENJEV = "openjev" # logits of one label token per option
+ LEV = "lev" # same as openjev, noul is read from a rating scale
+ KEV = "kev" # dot product of the hidden states of the last token and of one end token per option
+ NIMBLE = "nimble" # same as openjev, the prompt lists all the questions of the request
+ CLEF = "clef" # joint head over all questions, one score per option
+ PPLX_DECIDER = "pplx-decider" # same as openjev, label codes of 1 or 2 letters
+ VLLM_DECISION2 = "vllm-decision2" # head over the end token of each option and the last token, one score per option
+ VLLM_D3 = "vllm-d3" # same as pplx-decider, non-causal attention
+ LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type
+ LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout
+
+
+class VisionProjectorType:
+ GEMMA3 = "gemma3"
+ GEMMA3NV = "gemma3nv"
+ GEMMA3NA = "gemma3na"
+ GEMMA4V = "gemma4v"
+ GEMMA4A = "gemma4a"
+ GEMMA4UV = "gemma4uv" # "unified" variant
+ GEMMA4UA = "gemma4ua" # "unified" variant
+ PHI4 = "phi4"
+ IDEFICS3 = "idefics3"
+ PIXTRAL = "pixtral"
+ LLAMA4 = "llama4"
+ QWEN2VL = "qwen2vl_merger"
+ QWEN25VL = "qwen2.5vl_merger"
+ EXAONE4_5 = "exaone4_5"
+ QWEN3VL = "qwen3vl_merger"
+ LING3VL = "ling3vl"
+ STEP3VL = "step3vl"
+ ULTRAVOX = "ultravox"
+ INTERNVL = "internvl"
+ QWEN2A = "qwen2a" # audio
+ QWEN3A = "qwen3a" # audio
+ GLMA = "glma" # audio
+ QWEN25O = "qwen2.5o" # omni
+ VOXTRAL = "voxtral"
+ MERALION = "meralion" # audio: Whisper + gated MLP adaptor
+ LFM2 = "lfm2"
+ KIMIVL = "kimivl"
+ PADDLEOCR = "paddleocr"
+ KIMIK25 = "kimik25"
+ LIGHTONOCR = "lightonocr"
+ COGVLM = "cogvlm"
+ JANUS_PRO = "janus_pro"
+ DOTSOCR = "dots_ocr"
+ DOTS3NOTE_V = "dots3note_v"
+ DOTS3NOTE_A = "dots3note_a" # audio
+ DEEPSEEKOCR = "deepseekocr"
+ DEEPSEEKOCR2 = "deepseekocr2"
+ DEEPSEEK4V = "deepseek4v"
+ LFM2A = "lfm2a" # audio
+ MUSIC_FLAMINGO = "musicflamingo" # audio
+ GLM4V = "glm4v"
+ GLM5V = "glm5v"
+ YOUTUVL = "youtuvl"
+ NEMOTRON_V2_VL = "nemotron_v2_vl"
+ QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder
+ QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
+ POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder
+ POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder
+ HUNYUANVL = "hunyuanvl"
+ PARAKEET = "parakeet" # audio
+ MINIMAXM3 = "minimax_m3"
+ MINICPMV4_6 = "minicpmv4_6"
+ MINICPMV4_7 = "minicpmv4_7"
+ GRANITE_SPEECH = "granite_speech" # audio
+ MIMOVL = "mimovl"
+ MIMO_AUDIO = "mimo_audio"
+ GRANITE4_VISION = "granite4_vision"
+ MUSE_GLIMMER = "muse-glimmer"
+ COHERE2V = "cohere2v"
+ D1OMNI_V = "d1omni_v" # lfm2 vision, without separator tokens
+ D1OMNI_A = "d1omni_a" # lfm2a audio, with a residual block after the projector
+
+
+# Items here are (block size, type size)
+QK_K = 256
+GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = {
+ GGMLQuantizationType.F32: (1, 4),
+ GGMLQuantizationType.F16: (1, 2),
+ GGMLQuantizationType.Q4_0: (32, 2 + 16),
+ GGMLQuantizationType.Q4_1: (32, 2 + 2 + 16),
+ GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16),
+ GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16),
+ GGMLQuantizationType.Q8_0: (32, 2 + 32),
+ GGMLQuantizationType.Q8_1: (32, 2 + 2 + 32),
+ GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4),
+ GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12),
+ GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12),
+ GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),
+ GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),
+ GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8),
+ GGMLQuantizationType.IQ2_XXS: (256, 2 + QK_K // 4),
+ GGMLQuantizationType.IQ2_XS: (256, 2 + QK_K // 4 + QK_K // 32),
+ GGMLQuantizationType.IQ3_XXS: (256, 2 + QK_K // 4 + QK_K // 8),
+ GGMLQuantizationType.IQ1_S: (256, 2 + QK_K // 8 + QK_K // 16),
+ GGMLQuantizationType.IQ4_NL: (32, 2 + 16),
+ GGMLQuantizationType.IQ3_S: (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4),
+ GGMLQuantizationType.IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16),
+ GGMLQuantizationType.IQ4_XS: (256, 2 + 2 + QK_K // 2 + QK_K // 64),
+ GGMLQuantizationType.I8: (1, 1),
+ GGMLQuantizationType.I16: (1, 2),
+ GGMLQuantizationType.I32: (1, 4),
+ GGMLQuantizationType.I64: (1, 8),
+ GGMLQuantizationType.F64: (1, 8),
+ GGMLQuantizationType.IQ1_M: (256, QK_K // 8 + QK_K // 16 + QK_K // 32),
+ GGMLQuantizationType.BF16: (1, 2),
+ GGMLQuantizationType.TQ1_0: (256, 2 + 4 * 13),
+ GGMLQuantizationType.TQ2_0: (256, 2 + 64),
+ GGMLQuantizationType.MXFP4: (32, 1 + 16),
+ GGMLQuantizationType.NVFP4: (64, 4 + 32),
+ GGMLQuantizationType.Q1_0: (128, 2 + 16),
+ GGMLQuantizationType.Q2_0: (64, 2 + 16),
+}
+
+
+# Aliases for backward compatibility.
+
+# general
+KEY_GENERAL_ARCHITECTURE = Keys.General.ARCHITECTURE
+KEY_GENERAL_QUANTIZATION_VERSION = Keys.General.QUANTIZATION_VERSION
+KEY_GENERAL_ALIGNMENT = Keys.General.ALIGNMENT
+KEY_GENERAL_NAME = Keys.General.NAME
+KEY_GENERAL_AUTHOR = Keys.General.AUTHOR
+KEY_GENERAL_URL = Keys.General.URL
+KEY_GENERAL_DESCRIPTION = Keys.General.DESCRIPTION
+KEY_GENERAL_LICENSE = Keys.General.LICENSE
+KEY_GENERAL_SOURCE_URL = Keys.General.SOURCE_URL
+KEY_GENERAL_FILE_TYPE = Keys.General.FILE_TYPE
+
+# LLM
+KEY_VOCAB_SIZE = Keys.LLM.VOCAB_SIZE
+KEY_CONTEXT_LENGTH = Keys.LLM.CONTEXT_LENGTH
+KEY_EMBEDDING_LENGTH = Keys.LLM.EMBEDDING_LENGTH
+KEY_BLOCK_COUNT = Keys.LLM.BLOCK_COUNT
+KEY_FEED_FORWARD_LENGTH = Keys.LLM.FEED_FORWARD_LENGTH
+KEY_USE_PARALLEL_RESIDUAL = Keys.LLM.USE_PARALLEL_RESIDUAL
+KEY_TENSOR_DATA_LAYOUT = Keys.LLM.TENSOR_DATA_LAYOUT
+
+# attention
+KEY_ATTENTION_HEAD_COUNT = Keys.Attention.HEAD_COUNT
+KEY_ATTENTION_HEAD_COUNT_KV = Keys.Attention.HEAD_COUNT_KV
+KEY_ATTENTION_MAX_ALIBI_BIAS = Keys.Attention.MAX_ALIBI_BIAS
+KEY_ATTENTION_CLAMP_KQV = Keys.Attention.CLAMP_KQV
+KEY_ATTENTION_LAYERNORM_EPS = Keys.Attention.LAYERNORM_EPS
+KEY_ATTENTION_LAYERNORM_RMS_EPS = Keys.Attention.LAYERNORM_RMS_EPS
+
+# RoPE
+KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT
+KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE
+KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE
+KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR
+KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN
+KEY_ROPE_SCALING_FINETUNED = Keys.Rope.SCALING_FINETUNED
+
+# SSM
+KEY_SSM_CONV_KERNEL = Keys.SSM.CONV_KERNEL
+KEY_SSM_INNER_SIZE = Keys.SSM.INNER_SIZE
+KEY_SSM_STATE_SIZE = Keys.SSM.STATE_SIZE
+KEY_SSM_TIME_STEP_RANK = Keys.SSM.TIME_STEP_RANK
+KEY_SSM_GROUP_COUNT = Keys.SSM.GROUP_COUNT
+KEY_SSM_DT_B_C_RMS = Keys.SSM.DT_B_C_RMS
+
+# KDA
+KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM
+KEY_KDA_SAFE_GATE = Keys.KDA.SAFE_GATE
+KEY_KDA_GATE_LOWER_BOUND = Keys.KDA.GATE_LOWER_BOUND
+
+# tokenization
+KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL
+KEY_TOKENIZER_PRE = Keys.Tokenizer.PRE
+KEY_TOKENIZER_LIST = Keys.Tokenizer.LIST
+KEY_TOKENIZER_TOKEN_TYPE = Keys.Tokenizer.TOKEN_TYPE
+KEY_TOKENIZER_SCORES = Keys.Tokenizer.SCORES
+KEY_TOKENIZER_MERGES = Keys.Tokenizer.MERGES
+KEY_TOKENIZER_BOS_ID = Keys.Tokenizer.BOS_ID
+KEY_TOKENIZER_EOS_ID = Keys.Tokenizer.EOS_ID
+KEY_TOKENIZER_EOT_ID = Keys.Tokenizer.EOT_ID
+KEY_TOKENIZER_EOM_ID = Keys.Tokenizer.EOM_ID
+KEY_TOKENIZER_UNK_ID = Keys.Tokenizer.UNK_ID
+KEY_TOKENIZER_SEP_ID = Keys.Tokenizer.SEP_ID
+KEY_TOKENIZER_PAD_ID = Keys.Tokenizer.PAD_ID
+KEY_TOKENIZER_MASK_ID = Keys.Tokenizer.MASK_ID
+KEY_TOKENIZER_HF_JSON = Keys.Tokenizer.HF_JSON
+KEY_TOKENIZER_RWKV = Keys.Tokenizer.RWKV
+
+KEY_TOKENIZER_FIM_PRE_ID = Keys.Tokenizer.FIM_PRE_ID
+KEY_TOKENIZER_FIM_SUF_ID = Keys.Tokenizer.FIM_SUF_ID
+KEY_TOKENIZER_FIM_MID_ID = Keys.Tokenizer.FIM_MID_ID
+KEY_TOKENIZER_FIM_PAD_ID = Keys.Tokenizer.FIM_PAD_ID
+KEY_TOKENIZER_FIM_REP_ID = Keys.Tokenizer.FIM_REP_ID
+KEY_TOKENIZER_FIM_SEP_ID = Keys.Tokenizer.FIM_SEP_ID
+
+# deprecated
+KEY_TOKENIZER_PREFIX_ID = Keys.Tokenizer.PREFIX_ID
+KEY_TOKENIZER_SUFFIX_ID = Keys.Tokenizer.SUFFIX_ID
+KEY_TOKENIZER_MIDDLE_ID = Keys.Tokenizer.MIDDLE_ID
diff --git a/python/freetoken/third_party/gguf/freetoken.patch b/python/freetoken/third_party/gguf/freetoken.patch
new file mode 100644
index 00000000..b8016ff5
--- /dev/null
+++ b/python/freetoken/third_party/gguf/freetoken.patch
@@ -0,0 +1,23 @@
+--- a/gguf-py/gguf/__init__.py
++++ b/gguf-py/gguf/__init__.py
+@@ -1,9 +1,4 @@
+ from .constants import *
+ from .lazy import *
+ from .gguf_reader import *
+-from .gguf_writer import *
+ from .quants import *
+-from .tensor_mapping import *
+-from .vocab import *
+-from .utility import *
+-from .metadata import *
+--- a/gguf-py/gguf/gguf_reader.py
++++ b/gguf-py/gguf/gguf_reader.py
+@@ -21,7 +21,7 @@
+ # Allow running file in package as a script.
+ sys.path.insert(0, str(Path(__file__).parent.parent))
+
+-from gguf.constants import (
++from .constants import (
+ GGML_MAX_DIMS,
+ GGML_QUANT_SIZES,
+ GGUF_DEFAULT_ALIGNMENT,
diff --git a/python/freetoken/third_party/gguf/gguf_reader.py b/python/freetoken/third_party/gguf/gguf_reader.py
new file mode 100644
index 00000000..41eb6521
--- /dev/null
+++ b/python/freetoken/third_party/gguf/gguf_reader.py
@@ -0,0 +1,390 @@
+#
+# GGUF file reading/modification support. For API usage information,
+# please see the files scripts/ for some fairly simple examples.
+#
+from __future__ import annotations
+
+import logging
+import os
+import sys
+from collections import OrderedDict
+from typing import Any, Literal, NamedTuple, TypeVar, Union
+
+import numpy as np
+import numpy.typing as npt
+
+from .quants import quant_shape_to_byte_shape
+
+if __name__ == "__main__":
+ from pathlib import Path
+
+ # Allow running file in package as a script.
+ sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from .constants import (
+ GGML_MAX_DIMS,
+ GGML_QUANT_SIZES,
+ GGUF_DEFAULT_ALIGNMENT,
+ GGUF_MAGIC,
+ GGUF_VERSION,
+ GGMLQuantizationType,
+ GGUFValueType,
+ GGUFEndian,
+)
+
+# limits mirroring ggml/src/gguf.cpp (not part of gguf.h)
+GGUF_MAX_STRING_LENGTH = 1024 * 1024 * 1024
+GGUF_MAX_ARRAY_ELEMENTS = 1024 * 1024 * 1024
+
+logger = logging.getLogger(__name__)
+
+READER_SUPPORTED_VERSIONS = [2, GGUF_VERSION]
+
+
+class ReaderField(NamedTuple):
+ # Offset to start of this field.
+ offset: int
+
+ # Name of the field (not necessarily from file data).
+ name: str
+
+ # Data parts. Some types have multiple components, such as strings
+ # that consist of a length followed by the string data.
+ parts: list[npt.NDArray[Any]] = []
+
+ # Indexes into parts that we can call the actual data. For example
+ # an array of strings will be populated with indexes to the actual
+ # string data.
+ data: list[int] = [-1]
+
+ types: list[GGUFValueType] = []
+
+ def contents(self, index_or_slice: int | slice = slice(None)) -> Any:
+ if self.types:
+ to_string = lambda x: str(x.tobytes(), encoding='utf-8') # noqa: E731
+ main_type = self.types[0]
+
+ if main_type == GGUFValueType.ARRAY:
+ sub_type = self.types[-1]
+
+ if sub_type == GGUFValueType.STRING:
+ indices = self.data[index_or_slice]
+
+ if isinstance(index_or_slice, int):
+ return to_string(self.parts[indices]) # type: ignore
+ else:
+ return [to_string(self.parts[idx]) for idx in indices] # type: ignore
+ else:
+ # FIXME: When/if _get_field_parts() support multi-dimensional arrays, this must do so too
+
+ # Check if it's unsafe to perform slice optimization on data
+ # if any(True for idx in self.data if len(self.parts[idx]) != 1):
+ # optim_slice = slice(None)
+ # else:
+ # optim_slice = index_or_slice
+ # index_or_slice = slice(None)
+
+ # if isinstance(optim_slice, int):
+ # return self.parts[self.data[optim_slice]].tolist()[0]
+ # else:
+ # return [pv for idx in self.data[optim_slice] for pv in self.parts[idx].tolist()][index_or_slice]
+
+ if isinstance(index_or_slice, int):
+ return self.parts[self.data[index_or_slice]].tolist()[0]
+ else:
+ return [pv for idx in self.data[index_or_slice] for pv in self.parts[idx].tolist()]
+
+ if main_type == GGUFValueType.STRING:
+ return to_string(self.parts[-1])
+ else:
+ return self.parts[-1].tolist()[0]
+
+ return None
+
+
+class ReaderTensor(NamedTuple):
+ name: str
+ tensor_type: GGMLQuantizationType
+ shape: npt.NDArray[np.uint32]
+ n_elements: int
+ n_bytes: int
+ data_offset: int
+ data: npt.NDArray[Any]
+ field: ReaderField
+
+
+class GGUFReader:
+ # I - same as host, S - swapped
+ byte_order: Literal['I', 'S'] = 'I'
+ alignment: int = GGUF_DEFAULT_ALIGNMENT
+ data_offset: int
+
+ # Note: Internal helper, API may change.
+ gguf_scalar_to_np: dict[GGUFValueType, type[np.generic]] = {
+ GGUFValueType.UINT8: np.uint8,
+ GGUFValueType.INT8: np.int8,
+ GGUFValueType.UINT16: np.uint16,
+ GGUFValueType.INT16: np.int16,
+ GGUFValueType.UINT32: np.uint32,
+ GGUFValueType.INT32: np.int32,
+ GGUFValueType.FLOAT32: np.float32,
+ GGUFValueType.UINT64: np.uint64,
+ GGUFValueType.INT64: np.int64,
+ GGUFValueType.FLOAT64: np.float64,
+ GGUFValueType.BOOL: np.bool_,
+ }
+
+ def __init__(self, path: os.PathLike[str] | str, mode: Literal['r', 'r+', 'c'] = 'r'):
+ self.data = np.memmap(path, mode = mode)
+ offs = 0
+
+ # Check for GGUF magic
+ if self._get(offs, np.uint32, override_order = '<')[0] != GGUF_MAGIC:
+ raise ValueError('GGUF magic invalid')
+ offs += 4
+
+ # Check GGUF version
+ temp_version = self._get(offs, np.uint32)
+ if temp_version[0] & 65535 == 0:
+ # If we get 0 here that means it's (probably) a GGUF file created for
+ # the opposite byte order of the machine this script is running on.
+ self.byte_order = 'S'
+ temp_version = temp_version.view(temp_version.dtype.newbyteorder(self.byte_order))
+ version = temp_version[0]
+ if version not in READER_SUPPORTED_VERSIONS:
+ raise ValueError(f'Sorry, file appears to be version {version} which we cannot handle')
+ if sys.byteorder == "little":
+ # Host is little endian
+ host_endian = GGUFEndian.LITTLE
+ swapped_endian = GGUFEndian.BIG
+ else:
+ # Sorry PDP or other weird systems that don't use BE or LE.
+ host_endian = GGUFEndian.BIG
+ swapped_endian = GGUFEndian.LITTLE
+ self.endianess = swapped_endian if self.byte_order == "S" else host_endian
+ self.fields: OrderedDict[str, ReaderField] = OrderedDict()
+ self.tensors: list[ReaderTensor] = []
+ offs += self._push_field(ReaderField(offs, 'GGUF.version', [temp_version], [0], [GGUFValueType.UINT32]))
+
+ # Check tensor count and kv count
+ temp_counts = self._get(offs, np.uint64, 2)
+ offs += self._push_field(ReaderField(offs, 'GGUF.tensor_count', [temp_counts[:1]], [0], [GGUFValueType.UINT64]))
+ offs += self._push_field(ReaderField(offs, 'GGUF.kv_count', [temp_counts[1:]], [0], [GGUFValueType.UINT64]))
+ tensor_count, kv_count = temp_counts
+ if tensor_count > GGUF_MAX_ARRAY_ELEMENTS:
+ raise ValueError(f'Tensor count {tensor_count} exceeds maximum {GGUF_MAX_ARRAY_ELEMENTS}')
+ if kv_count > GGUF_MAX_ARRAY_ELEMENTS:
+ raise ValueError(f'KV count {kv_count} exceeds maximum {GGUF_MAX_ARRAY_ELEMENTS}')
+ offs = self._build_fields(offs, kv_count)
+
+ # Build Tensor Info Fields
+ offs, tensors_fields = self._build_tensor_info(offs, tensor_count)
+ new_align = self.fields.get('general.alignment')
+ if new_align is not None:
+ if new_align.types != [GGUFValueType.UINT32]:
+ raise ValueError('Bad type for general.alignment field')
+ self.alignment = new_align.parts[-1][0]
+ # Ensure alignment is a non-zero power of two
+ if self.alignment == 0 or (self.alignment & (self.alignment - 1)) != 0:
+ raise ValueError('Invalid alignment: must be a non-zero power of two')
+ padding = offs % self.alignment
+ if padding != 0:
+ offs += self.alignment - padding
+ self.data_offset = offs
+ self._build_tensors(offs, tensors_fields)
+
+ _DT = TypeVar('_DT', bound = npt.DTypeLike)
+
+ # Fetch a key/value metadata field by key.
+ def get_field(self, key: str) -> Union[ReaderField, None]:
+ return self.fields.get(key, None)
+
+ # Fetch a tensor from the list by index.
+ def get_tensor(self, idx: int) -> ReaderTensor:
+ return self.tensors[idx]
+
+ def _get(
+ self, offset: int, dtype: npt.DTypeLike, count: int = 1, override_order: None | Literal['I', 'S', '<'] = None,
+ ) -> npt.NDArray[Any]:
+ count = int(count)
+ itemsize = int(np.empty([], dtype = dtype).itemsize)
+ end_offs = offset + itemsize * count
+ arr = self.data[offset:end_offs].view(dtype=dtype)[:count]
+ return arr.view(arr.dtype.newbyteorder(self.byte_order if override_order is None else override_order))
+
+ def _push_field(self, field: ReaderField, skip_sum: bool = False) -> int:
+ if field.name in self.fields:
+ # TODO: add option to make this a warning and accept duplicate keys like below
+ raise KeyError(f'Duplicate {field.name} already in list at offset {field.offset}')
+
+ # logger.warning(f'Duplicate key {field.name} at offset {field.offset}')
+ # self.fields[field.name + '_{}'.format(field.offset)] = field
+ else:
+ self.fields[field.name] = field
+ return 0 if skip_sum else sum(int(part.nbytes) for part in field.parts)
+
+ def _get_str(self, offset: int) -> tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint8]]:
+ slen = self._get(offset, np.uint64)
+ if int(slen[0]) > GGUF_MAX_STRING_LENGTH:
+ raise ValueError(f'String length {int(slen[0])} exceeds maximum {GGUF_MAX_STRING_LENGTH}')
+ if offset + 8 + int(slen[0]) > self.data.nbytes:
+ raise ValueError(f'String length {int(slen[0])} exceeds remaining file size {self.data.nbytes - offset - 8}')
+ return slen, self._get(offset + 8, np.uint8, slen[0])
+
+ def _get_field_parts(
+ self, orig_offs: int, raw_type: int,
+ ) -> tuple[int, list[npt.NDArray[Any]], list[int], list[GGUFValueType]]:
+ offs = orig_offs
+ types: list[GGUFValueType] = []
+ gtype = GGUFValueType(raw_type)
+ types.append(gtype)
+ # Handle strings.
+ if gtype == GGUFValueType.STRING:
+ sparts: list[npt.NDArray[Any]] = list(self._get_str(offs))
+ size = sum(int(part.nbytes) for part in sparts)
+ return size, sparts, [1], types
+ # Check if it's a simple scalar type.
+ nptype = self.gguf_scalar_to_np.get(gtype)
+ if nptype is not None:
+ val = self._get(offs, nptype)
+ return int(val.nbytes), [val], [0], types
+ # Handle arrays.
+ if gtype == GGUFValueType.ARRAY:
+ raw_itype = self._get(offs, np.uint32)
+ offs += int(raw_itype.nbytes)
+ alen = self._get(offs, np.uint64)
+ if int(alen[0]) > GGUF_MAX_ARRAY_ELEMENTS:
+ raise ValueError(f'Array length {int(alen[0])} exceeds maximum {GGUF_MAX_ARRAY_ELEMENTS}')
+ offs += int(alen.nbytes)
+ aparts: list[npt.NDArray[Any]] = [raw_itype, alen]
+ data_idxs: list[int] = []
+ # FIXME: Handle multi-dimensional arrays properly instead of flattening
+ for idx in range(alen[0]):
+ curr_size, curr_parts, curr_idxs, curr_types = self._get_field_parts(offs, raw_itype[0])
+ if idx == 0:
+ types += curr_types
+ idxs_offs = len(aparts)
+ aparts += curr_parts
+ data_idxs += (idx + idxs_offs for idx in curr_idxs)
+ offs += curr_size
+ return offs - orig_offs, aparts, data_idxs, types
+ # We can't deal with this one.
+ raise ValueError(f'Unknown/unhandled field type {gtype}')
+
+ def _get_tensor_info_field(self, orig_offs: int) -> ReaderField:
+ offs = orig_offs
+
+ # Get Tensor Name
+ name_len, name_data = self._get_str(offs)
+ offs += int(name_len.nbytes + name_data.nbytes)
+
+ # Get Tensor Dimensions Count
+ n_dims = self._get(offs, np.uint32)
+ offs += int(n_dims.nbytes)
+ if n_dims[0] > GGML_MAX_DIMS:
+ raise ValueError(f'Tensor dimensions count {n_dims[0]} exceeds GGML_MAX_DIMS ({GGML_MAX_DIMS})')
+
+ # Get Tensor Dimension Array
+ dims = self._get(offs, np.uint64, n_dims[0])
+ offs += int(dims.nbytes)
+
+ # Get Tensor Encoding Scheme Type
+ raw_dtype = self._get(offs, np.uint32)
+ offs += int(raw_dtype.nbytes)
+
+ # Get Tensor Offset
+ offset_tensor = self._get(offs, np.uint64)
+ offs += int(offset_tensor.nbytes)
+
+ return ReaderField(
+ orig_offs,
+ str(bytes(name_data), encoding = 'utf-8'),
+ [name_len, name_data, n_dims, dims, raw_dtype, offset_tensor],
+ [1, 3, 4, 5],
+ )
+
+ def _build_fields(self, offs: int, count: int) -> int:
+ for _ in range(count):
+ orig_offs = offs
+ kv_klen, kv_kdata = self._get_str(offs)
+ offs += int(kv_klen.nbytes + kv_kdata.nbytes)
+ raw_kv_type = self._get(offs, np.uint32)
+ offs += int(raw_kv_type.nbytes)
+ parts: list[npt.NDArray[Any]] = [kv_klen, kv_kdata, raw_kv_type]
+ idxs_offs = len(parts)
+ field_size, field_parts, field_idxs, field_types = self._get_field_parts(offs, raw_kv_type[0])
+ parts += field_parts
+ self._push_field(ReaderField(
+ orig_offs,
+ str(bytes(kv_kdata), encoding = 'utf-8'),
+ parts,
+ [idx + idxs_offs for idx in field_idxs],
+ field_types,
+ ), skip_sum = True)
+ offs += field_size
+ return offs
+
+ def _build_tensor_info(self, offs: int, count: int) -> tuple[int, list[ReaderField]]:
+ tensor_fields = []
+ for _ in range(count):
+ field = self._get_tensor_info_field(offs)
+ offs += sum(int(part.nbytes) for part in field.parts)
+ tensor_fields.append(field)
+ return offs, tensor_fields
+
+ def _build_tensors(self, start_offs: int, fields: list[ReaderField]) -> None:
+ tensors = []
+ tensor_names = set() # keep track of name to prevent duplicated tensors
+ for field in fields:
+ _name_len, name_data, _n_dims, dims, raw_dtype, offset_tensor = field.parts
+ # check if there's any tensor having same name already in the list
+ tensor_name = str(bytes(name_data), encoding = 'utf-8')
+ if tensor_name in tensor_names:
+ raise ValueError(f'Found duplicated tensor with name {tensor_name}')
+ tensor_names.add(tensor_name)
+ ggml_type = GGMLQuantizationType(raw_dtype[0])
+ # use Python ints: np.prod on uint64 wraps silently on overflow
+ n_elems = 1
+ for dim in dims.tolist():
+ n_elems *= int(dim)
+ np_dims = tuple(reversed(dims.tolist()))
+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]
+ n_bytes = n_elems * type_size // block_size
+ data_offs = int(start_offs + offset_tensor[0])
+ item_type: npt.DTypeLike
+ if ggml_type == GGMLQuantizationType.F16:
+ item_count = n_elems
+ item_type = np.float16
+ elif ggml_type == GGMLQuantizationType.F32:
+ item_count = n_elems
+ item_type = np.float32
+ elif ggml_type == GGMLQuantizationType.F64:
+ item_count = n_elems
+ item_type = np.float64
+ elif ggml_type == GGMLQuantizationType.I8:
+ item_count = n_elems
+ item_type = np.int8
+ elif ggml_type == GGMLQuantizationType.I16:
+ item_count = n_elems
+ item_type = np.int16
+ elif ggml_type == GGMLQuantizationType.I32:
+ item_count = n_elems
+ item_type = np.int32
+ elif ggml_type == GGMLQuantizationType.I64:
+ item_count = n_elems
+ item_type = np.int64
+ else:
+ item_count = n_bytes
+ item_type = np.uint8
+ np_dims = quant_shape_to_byte_shape(np_dims, ggml_type)
+ tensors.append(ReaderTensor(
+ name = tensor_name,
+ tensor_type = ggml_type,
+ shape = dims,
+ n_elements = n_elems,
+ n_bytes = n_bytes,
+ data_offset = data_offs,
+ data = self._get(data_offs, item_type, item_count).reshape(np_dims),
+ field = field,
+ ))
+ self.tensors = tensors
diff --git a/python/freetoken/third_party/gguf/lazy.py b/python/freetoken/third_party/gguf/lazy.py
new file mode 100644
index 00000000..a39f2232
--- /dev/null
+++ b/python/freetoken/third_party/gguf/lazy.py
@@ -0,0 +1,293 @@
+from __future__ import annotations
+from abc import ABC, ABCMeta, abstractmethod
+
+import logging
+from typing import Any, Callable
+
+import numpy as np
+from numpy.typing import DTypeLike
+
+
+logger = logging.getLogger(__name__)
+
+
+class LazyMeta(ABCMeta):
+
+ def __new__(cls, name: str, bases: tuple[type, ...], namespace: dict[str, Any], **kwargs):
+ def __getattr__(self, name: str) -> Any:
+ meta_attr = getattr(self._meta, name)
+ if callable(meta_attr):
+ return type(self)._wrap_fn(
+ (lambda s, *args, **kwargs: getattr(s, name)(*args, **kwargs)),
+ use_self=self,
+ )
+ elif isinstance(meta_attr, self._tensor_type):
+ # e.g. self.T with torch.Tensor should still be wrapped
+ return type(self)._wrap_fn(lambda s: getattr(s, name))(self)
+ else:
+ # no need to wrap non-tensor properties,
+ # and they likely don't depend on the actual contents of the tensor
+ return meta_attr
+
+ namespace["__getattr__"] = __getattr__
+
+ # need to make a builder for the wrapped wrapper to copy the name,
+ # or else it fails with very cryptic error messages,
+ # because somehow the same string would end up in every closures
+ def mk_wrap(op_name: str, *, meta_noop: bool = False):
+ # need to wrap the wrapper to get self
+ def wrapped_special_op(self, *args, **kwargs):
+ return type(self)._wrap_fn(
+ getattr(type(self)._tensor_type, op_name),
+ meta_noop=meta_noop,
+ )(self, *args, **kwargs)
+ return wrapped_special_op
+
+ # special methods bypass __getattr__, so they need to be added manually
+ # ref: https://docs.python.org/3/reference/datamodel.html#special-lookup
+ # NOTE: doing this from a metaclass is very convenient
+ # TODO: make this even more comprehensive
+ for binary_op in (
+ "lt", "le", "eq", "ne", "ge", "gt",
+ "add", "and", "floordiv", "lshift", "mod", "mul", "matmul",
+ "or", "pow", "rshift", "sub", "truediv", "xor",
+ "iadd", "iand", "ifloordiv", "ilshift", "imod", "imul", "ior", "irshift", "isub", "ixor",
+ "radd", "rand", "rfloordiv", "rmul", "ror", "rpow", "rsub", "rtruediv", "rxor",
+ ):
+ attr_name = f"__{binary_op}__"
+ # evaluation on the meta tensor is needed in case there's broadcasting
+ namespace[attr_name] = mk_wrap(attr_name, meta_noop=False)
+
+ for unary_op in ("not", "abs", "invert", "neg", "pos"):
+ attr_name = f"__{unary_op}__"
+ # the result of these operators usually has the same shape and dtype as the input,
+ # so evaluation on the meta tensor can be skipped.
+ namespace[attr_name] = mk_wrap(attr_name, meta_noop=True)
+
+ for special_op in (
+ "getitem", "setitem", "len",
+ ):
+ attr_name = f"__{special_op}__"
+ namespace[attr_name] = mk_wrap(attr_name, meta_noop=False)
+
+ return super().__new__(cls, name, bases, namespace, **kwargs)
+
+
+# Tree of lazy tensors
+class LazyBase(ABC, metaclass=LazyMeta):
+ _tensor_type: type
+ _meta: Any
+ _data: Any | None
+ _args: tuple
+ _kwargs: dict[str, Any]
+ _func: Callable[[Any], Any] | None
+
+ def __init__(self, *, meta: Any, data: Any | None = None, args: tuple = (), kwargs: dict[str, Any] | None = None, func: Callable[[Any], Any] | None = None):
+ super().__init__()
+ self._meta = meta
+ self._data = data
+ self._args = args
+ self._kwargs = kwargs if kwargs is not None else {}
+ self._func = func
+ assert self._func is not None or self._data is not None
+
+ def __init_subclass__(cls) -> None:
+ if "_tensor_type" not in cls.__dict__:
+ raise TypeError(f"property '_tensor_type' must be defined for {cls!r}")
+ return super().__init_subclass__()
+
+ @staticmethod
+ def _recurse_apply(o: Any, fn: Callable[[Any], Any]) -> Any:
+ # TODO: dict and set
+ if isinstance(o, (list, tuple)):
+ L = []
+ for item in o:
+ L.append(LazyBase._recurse_apply(item, fn))
+ if isinstance(o, tuple):
+ L = tuple(L)
+ return L
+ elif isinstance(o, LazyBase):
+ return fn(o)
+ else:
+ return o
+
+ @classmethod
+ def _wrap_fn(cls, fn: Callable, *, use_self: LazyBase | None = None, meta_noop: bool | DTypeLike | tuple[DTypeLike, Callable[[tuple[int, ...]], tuple[int, ...]]] = False) -> Callable[[Any], Any]:
+ def wrapped_fn(*args, **kwargs):
+ if kwargs is None:
+ kwargs = {}
+ args = ((use_self,) if use_self is not None else ()) + args
+
+ meta_args = LazyBase._recurse_apply(args, lambda t: t._meta)
+ # TODO: maybe handle tensors in kwargs too
+
+ if isinstance(meta_noop, bool) and not meta_noop:
+ try:
+ res = fn(*meta_args, **kwargs)
+ except NotImplementedError:
+ # running some operations on PyTorch's Meta tensors can cause this exception
+ res = None
+ else:
+ # some operators don't need to actually run on the meta tensors
+ assert len(args) > 0
+ res = args[0]
+ assert isinstance(res, cls)
+ res = res._meta
+ # allow operations to override the dtype and shape
+ if meta_noop is not True:
+ if isinstance(meta_noop, tuple):
+ dtype, shape = meta_noop
+ assert callable(shape)
+ res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape)) # ty: ignore[call-top-callable]
+ else:
+ res = cls.meta_with_dtype_and_shape(meta_noop, res.shape)
+
+ if isinstance(res, cls._tensor_type):
+ return cls(meta=cls.eager_to_meta(res), args=args, kwargs=kwargs, func=fn)
+ elif isinstance(res, tuple) and all(isinstance(t, cls._tensor_type) for t in res):
+ # share the evaluation between lazy tuple elements
+ shared_args: list = [args, None]
+
+ def eager_tuple_element(a: list[Any], i: int = 0, /, **kw) -> LazyBase:
+ assert len(a) == 2
+ if a[1] is None:
+ a[1] = fn(*a[0], **kw)
+ return a[1][i]
+ return tuple(cls(meta=cls.eager_to_meta(res[i]), args=(shared_args, i), kwargs=kwargs, func=eager_tuple_element) for i in range(len(res)))
+ else:
+ del res # not needed
+ # non-tensor return likely relies on the contents of the args
+ # (e.g. the result of torch.equal)
+ eager_args = cls.to_eager(args)
+ return fn(*eager_args, **kwargs)
+ return wrapped_fn
+
+ @classmethod
+ def to_eager(cls, t: Any) -> Any:
+ def simple_to_eager(_t: LazyBase) -> Any:
+ if _t._data is not None:
+ return _t._data
+
+ # NOTE: there's a recursion limit in Python (usually 1000)
+
+ assert _t._func is not None
+ _t._args = cls._recurse_apply(_t._args, simple_to_eager)
+ _t._data = _t._func(*_t._args, **_t._kwargs)
+ # sanity check
+ assert _t._data is not None
+ assert _t._data.dtype == _t._meta.dtype
+ assert _t._data.shape == _t._meta.shape
+
+ return _t._data
+
+ # recurse into lists and/or tuples, keeping their structure
+ return cls._recurse_apply(t, simple_to_eager)
+
+ @classmethod
+ def eager_to_meta(cls, t: Any) -> Any:
+ return cls.meta_with_dtype_and_shape(t.dtype, t.shape)
+
+ # must be overridden, meta tensor init is backend-specific
+ @classmethod
+ @abstractmethod
+ def meta_with_dtype_and_shape(cls, dtype: Any, shape: Any) -> Any: pass
+
+ @classmethod
+ def from_eager(cls, t: Any) -> Any:
+ if type(t) is cls:
+ # already lazy
+ return t
+ elif isinstance(t, cls._tensor_type):
+ return cls(meta=cls.eager_to_meta(t), data=t)
+ else:
+ return TypeError(f"{type(t)!r} is not compatible with {cls._tensor_type!r}")
+
+
+class LazyNumpyTensor(LazyBase):
+ _tensor_type = np.ndarray
+
+ shape: tuple[int, ...] # Makes the type checker happy in quants.py
+
+ @classmethod
+ def meta_with_dtype_and_shape(cls, dtype: DTypeLike, shape: tuple[int, ...]) -> np.ndarray[Any, Any]:
+ # The initial idea was to use np.nan as the fill value,
+ # but non-float types like np.int16 can't use that.
+ # So zero it is.
+ cheat = np.zeros(1, dtype)
+ return np.lib.stride_tricks.as_strided(cheat, shape, (0 for _ in shape))
+
+ def astype(self, dtype, *args, **kwargs):
+ meta = type(self).meta_with_dtype_and_shape(dtype, self._meta.shape)
+ full_args = (self, dtype,) + args
+ return type(self)(meta=meta, args=full_args, kwargs=kwargs, func=(lambda a, *args, **kwargs: a.astype(*args, **kwargs)))
+
+ def tofile(self, *args, **kwargs):
+ eager = LazyNumpyTensor.to_eager(self)
+ return eager.tofile(*args, **kwargs)
+
+ # TODO: __array_function__
+
+
+# Tensor written to file one row-chunk at a time
+class LazyChunkedTensor:
+
+ def __init__(
+ self, chunks: list[Callable[[], np.ndarray]], shape: tuple[int, ...], dtype: DTypeLike,
+ qtype: Any = None, byteswap: bool = False,
+ ):
+ self._chunks = chunks
+ self._qtype = qtype
+ self._byteswap = byteswap
+ self.shape = tuple(shape)
+ self.dtype = np.dtype(dtype)
+
+ @property
+ def nbytes(self) -> int:
+ n = self.dtype.itemsize
+ for d in self.shape:
+ n *= d
+ return n
+
+ def numpy(self) -> LazyChunkedTensor:
+ return self
+
+ def __array__(self, *args, **kwargs):
+ # numpy would otherwise make a 1-element object array of self, and write 8 bytes
+ raise TypeError("LazyChunkedTensor cannot become an ndarray, it is written in chunks")
+
+ def quantize(self, qtype: Any) -> LazyChunkedTensor:
+ from .constants import GGMLQuantizationType
+ from .quants import QuantError, quant_shape_to_byte_shape
+
+ if qtype == GGMLQuantizationType.F32:
+ shape, dtype = self.shape, np.dtype(np.float32)
+ elif qtype == GGMLQuantizationType.F16:
+ shape, dtype = self.shape, np.dtype(np.float16)
+ else:
+ try:
+ shape, dtype = quant_shape_to_byte_shape(self.shape, qtype), np.dtype(np.uint8)
+ except ValueError as e:
+ # raised here and not per chunk, so callers can still fall back to F16
+ raise QuantError(str(e)) from e
+ return LazyChunkedTensor(self._chunks, shape, dtype, qtype, self._byteswap)
+
+ def byteswap(self, inplace: bool = False) -> LazyChunkedTensor:
+ if inplace:
+ raise NotImplementedError("a chunked tensor cannot be byteswapped in place")
+ return LazyChunkedTensor(self._chunks, self.shape, self.dtype, self._qtype, not self._byteswap)
+
+ def tofile(self, *args, **kwargs) -> None:
+ from .quants import quantize
+
+ written = 0
+ for load_chunk in self._chunks:
+ chunk = load_chunk()
+ if self._qtype is not None:
+ # exact only because chunks split on rows, and blocks never cross one
+ chunk = quantize(chunk, self._qtype)
+ if self._byteswap:
+ chunk = chunk.byteswap(inplace=False)
+ chunk.tofile(*args, **kwargs)
+ written += chunk.nbytes
+ del chunk
+ assert written == self.nbytes, f"chunked tensor wrote {written} bytes, expected {self.nbytes}"
diff --git a/python/freetoken/third_party/gguf/quants.py b/python/freetoken/third_party/gguf/quants.py
new file mode 100644
index 00000000..80966b6e
--- /dev/null
+++ b/python/freetoken/third_party/gguf/quants.py
@@ -0,0 +1,1378 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Sequence
+from math import log2, ceil
+
+from numpy.typing import DTypeLike
+
+from .constants import GGML_QUANT_SIZES, GGMLQuantizationType, QK_K
+from .lazy import LazyNumpyTensor
+
+import numpy as np
+
+
+def quant_shape_to_byte_shape(shape: Sequence[int], quant_type: GGMLQuantizationType) -> tuple[int, ...]:
+ block_size, type_size = GGML_QUANT_SIZES[quant_type]
+ if shape[-1] % block_size != 0:
+ raise ValueError(f"Quantized tensor row size ({shape[-1]}) is not a multiple of {quant_type.name} block size ({block_size})")
+ return (*shape[:-1], shape[-1] // block_size * type_size)
+
+
+def quant_shape_from_byte_shape(shape: Sequence[int], quant_type: GGMLQuantizationType) -> tuple[int, ...]:
+ block_size, type_size = GGML_QUANT_SIZES[quant_type]
+ if shape[-1] % type_size != 0:
+ raise ValueError(f"Quantized tensor bytes per row ({shape[-1]}) is not a multiple of {quant_type.name} type size ({type_size})")
+ return (*shape[:-1], shape[-1] // type_size * block_size)
+
+
+# This is faster than np.vectorize and np.apply_along_axis because it works on more than one row at a time
+def _apply_over_grouped_rows(func: Callable[[np.ndarray], np.ndarray], arr: np.ndarray, otype: DTypeLike, oshape: tuple[int, ...]) -> np.ndarray:
+ rows = arr.reshape((-1, arr.shape[-1]))
+ assert len(rows.shape)
+ osize = 1
+ for dim in oshape:
+ osize *= dim
+ out = np.empty(shape=osize, dtype=otype)
+ # compute over groups of 16 rows (arbitrary, but seems good for performance)
+ n_groups = (rows.shape[0] // 16) or 1
+ np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out)
+ return out.reshape(oshape)
+
+
+# round away from zero
+# ref: https://stackoverflow.com/a/59143326/22827863
+def np_roundf(n: np.ndarray) -> np.ndarray:
+ a = abs(n)
+ floored = np.floor(a)
+ b = floored + np.floor(2 * (a - floored))
+ return np.sign(n) * b
+
+
+class QuantError(Exception): ...
+
+
+_type_traits: dict[GGMLQuantizationType, type[__Quant]] = {}
+
+
+def quantize(data: np.ndarray, qtype: GGMLQuantizationType) -> np.ndarray:
+ if qtype == GGMLQuantizationType.F32:
+ return data.astype(np.float32, copy=False)
+ elif qtype == GGMLQuantizationType.F16:
+ return data.astype(np.float16, copy=False)
+ elif (q := _type_traits.get(qtype)) is not None:
+ return q.quantize(data)
+ else:
+ raise NotImplementedError(f"Quantization for {qtype.name} is not yet implemented")
+
+
+def dequantize(data: np.ndarray, qtype: GGMLQuantizationType) -> np.ndarray:
+ if qtype == GGMLQuantizationType.F32:
+ return data.view(np.float32)
+ elif qtype == GGMLQuantizationType.F16:
+ return data.view(np.float16).astype(np.float32)
+ elif (q := _type_traits.get(qtype)) is not None:
+ return q.dequantize(data)
+ else:
+ raise NotImplementedError(f"Dequantization for {qtype.name} is not yet implemented")
+
+
+class __Quant(ABC):
+ qtype: GGMLQuantizationType
+ block_size: int
+ type_size: int
+
+ grid: np.ndarray[Any, np.dtype[np.float32]] | None = None
+ grid_shape: tuple[int, int] = (0, 0)
+ grid_map: tuple[int | float, ...] = ()
+ grid_hex: bytes | None = None
+
+ def __init__(self):
+ return TypeError("Quant conversion classes can't have instances")
+
+ def __init_subclass__(cls, qtype: GGMLQuantizationType) -> None:
+ cls.qtype = qtype
+ cls.block_size, cls.type_size = GGML_QUANT_SIZES[qtype]
+ cls.__quantize_lazy: Any = LazyNumpyTensor._wrap_fn(
+ cls.__quantize_array,
+ meta_noop=(np.uint8, cls.__shape_to_bytes)
+ )
+ cls.__dequantize_lazy: Any = LazyNumpyTensor._wrap_fn(
+ cls.__dequantize_array,
+ meta_noop=(np.float32, cls.__shape_from_bytes)
+ )
+ assert qtype not in _type_traits
+ _type_traits[qtype] = cls
+
+ @classmethod
+ def init_grid(cls):
+ if cls.grid is not None or cls.grid_hex is None:
+ return
+
+ bits_per_elem = ceil(log2(len(cls.grid_map)))
+ assert bits_per_elem != 0, cls.qtype.name
+ elems_per_byte = 8 // bits_per_elem
+
+ grid = np.frombuffer(cls.grid_hex, dtype=np.uint8)
+ # decode hexadecimal chars from grid
+ grid = grid.reshape((-1, 2))
+ grid = (np.where(grid > 0x40, grid + 9, grid) & 0x0F) << np.array([4, 0], dtype=np.uint8).reshape((1, 2))
+ grid = grid[..., 0] | grid[..., 1]
+ # unpack the grid values
+ grid = grid.reshape((-1, 1)) >> np.array([i for i in range(0, 8, 8 // elems_per_byte)], dtype=np.uint8).reshape((1, elems_per_byte))
+ grid = (grid & ((1 << bits_per_elem) - 1)).reshape((-1, 1))
+ grid_map = np.array(cls.grid_map, dtype=np.float32).reshape((1, -1))
+ grid = np.take_along_axis(grid_map, grid, axis=-1)
+ cls.grid = grid.reshape((1, 1, *cls.grid_shape))
+
+ @classmethod
+ @abstractmethod
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ raise NotImplementedError
+
+ @classmethod
+ @abstractmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ raise NotImplementedError
+
+ @classmethod
+ def quantize_rows(cls, rows: np.ndarray) -> np.ndarray:
+ rows = rows.astype(np.float32, copy=False)
+ shape = rows.shape
+ n_blocks = rows.size // cls.block_size
+ blocks = rows.reshape((n_blocks, cls.block_size))
+ blocks = cls.quantize_blocks(blocks)
+ assert blocks.dtype == np.uint8
+ assert blocks.shape[-1] == cls.type_size
+ return blocks.reshape(cls.__shape_to_bytes(shape))
+
+ @classmethod
+ def dequantize_rows(cls, rows: np.ndarray) -> np.ndarray:
+ rows = rows.view(np.uint8)
+ shape = rows.shape
+ n_blocks = rows.size // cls.type_size
+ blocks = rows.reshape((n_blocks, cls.type_size))
+ blocks = cls.dequantize_blocks(blocks)
+ assert blocks.dtype == np.float32
+ assert blocks.shape[-1] == cls.block_size
+ return blocks.reshape(cls.__shape_from_bytes(shape))
+
+ @classmethod
+ def __shape_to_bytes(cls, shape: Sequence[int]):
+ return quant_shape_to_byte_shape(shape, cls.qtype)
+
+ @classmethod
+ def __shape_from_bytes(cls, shape: Sequence[int]):
+ return quant_shape_from_byte_shape(shape, cls.qtype)
+
+ @classmethod
+ def __quantize_array(cls, array: np.ndarray) -> np.ndarray:
+ return _apply_over_grouped_rows(cls.quantize_rows, arr=array, otype=np.uint8, oshape=cls.__shape_to_bytes(array.shape))
+
+ @classmethod
+ def __dequantize_array(cls, array: np.ndarray) -> np.ndarray:
+ cls.init_grid()
+ return _apply_over_grouped_rows(cls.dequantize_rows, arr=array, otype=np.float32, oshape=cls.__shape_from_bytes(array.shape))
+
+ @classmethod
+ def __quantize_lazy(cls, lazy_tensor: LazyNumpyTensor, /) -> Any:
+ pass
+
+ @classmethod
+ def __dequantize_lazy(cls, lazy_tensor: LazyNumpyTensor, /) -> Any:
+ pass
+
+ @classmethod
+ def can_quantize(cls, tensor: np.ndarray | LazyNumpyTensor) -> bool:
+ return tensor.shape[-1] % cls.block_size == 0
+
+ @classmethod
+ def quantize(cls, tensor: np.ndarray | LazyNumpyTensor) -> np.ndarray:
+ if not cls.can_quantize(tensor):
+ raise QuantError(f"Can't quantize tensor with shape {tensor.shape} to {cls.qtype.name}")
+ if isinstance(tensor, LazyNumpyTensor):
+ return cls.__quantize_lazy(tensor)
+ else:
+ return cls.__quantize_array(tensor)
+
+ @classmethod
+ def dequantize(cls, tensor: np.ndarray | LazyNumpyTensor) -> np.ndarray:
+ if isinstance(tensor, LazyNumpyTensor):
+ return cls.__dequantize_lazy(tensor)
+ else:
+ return cls.__dequantize_array(tensor)
+
+
+class BF16(__Quant, qtype=GGMLQuantizationType.BF16):
+ @classmethod
+ # same as ggml_compute_fp32_to_bf16 in ggml-impl.h
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n = blocks.view(np.uint32)
+ # force nan to quiet
+ n = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n)
+ # round to nearest even
+ n = (np.uint64(n) + (0x7fff + ((n >> 16) & 1))) >> 16
+ return n.astype(np.uint16).view(np.uint8)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ return (blocks.view(np.int16).astype(np.int32) << 16).view(np.float32)
+
+
+class Q4_0(__Quant, qtype=GGMLQuantizationType.Q4_0):
+ @classmethod
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ imax = abs(blocks).argmax(axis=-1, keepdims=True)
+ max = np.take_along_axis(blocks, imax, axis=-1)
+
+ d = max / -8
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ qs = np.trunc((blocks * id) + np.float32(8.5), dtype=np.float32).astype(np.uint8).clip(0, 15)
+
+ qs = qs.reshape((n_blocks, 2, cls.block_size // 2))
+ qs = qs[..., 0, :] | (qs[..., 1, :] << np.uint8(4))
+
+ d = d.astype(np.float16).view(np.uint8)
+
+ return np.concatenate([d, qs], axis=-1)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, qs = np.hsplit(blocks, [2])
+
+ d = d.view(np.float16).astype(np.float32)
+
+ qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+ qs = (qs & np.uint8(0x0F)).reshape((n_blocks, -1)).astype(np.int8) - np.int8(8)
+
+ return (d * qs.astype(np.float32))
+
+
+class Q4_1(__Quant, qtype=GGMLQuantizationType.Q4_1):
+ @classmethod
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ max = blocks.max(axis=-1, keepdims=True)
+ min = blocks.min(axis=-1, keepdims=True)
+
+ d = (max - min) / 15
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ qs = np.trunc((blocks - min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 15)
+
+ qs = qs.reshape((n_blocks, 2, cls.block_size // 2))
+ qs = qs[..., 0, :] | (qs[..., 1, :] << np.uint8(4))
+
+ d = d.astype(np.float16).view(np.uint8)
+ m = min.astype(np.float16).view(np.uint8)
+
+ return np.concatenate([d, m, qs], axis=-1)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ m, qs = np.hsplit(rest, [2])
+
+ d = d.view(np.float16).astype(np.float32)
+ m = m.view(np.float16).astype(np.float32)
+
+ qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+ qs = (qs & np.uint8(0x0F)).reshape((n_blocks, -1)).astype(np.float32)
+
+ return (d * qs) + m
+
+
+class Q5_0(__Quant, qtype=GGMLQuantizationType.Q5_0):
+ @classmethod
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ imax = abs(blocks).argmax(axis=-1, keepdims=True)
+ max = np.take_along_axis(blocks, imax, axis=-1)
+
+ d = max / -16
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ q = np.trunc((blocks * id) + np.float32(16.5), dtype=np.float32).astype(np.uint8).clip(0, 31)
+
+ qs = q.reshape((n_blocks, 2, cls.block_size // 2))
+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))
+
+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)
+
+ d = d.astype(np.float16).view(np.uint8)
+
+ return np.concatenate([d, qh, qs], axis=-1)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ qh, qs = np.hsplit(rest, [4])
+
+ d = d.view(np.float16).astype(np.float32)
+ qh = qh.view(np.uint32)
+
+ qh = qh.reshape((n_blocks, 1)) >> np.array([i for i in range(32)], dtype=np.uint32).reshape((1, 32))
+ ql = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+ qh = (qh & np.uint32(0x01)).astype(np.uint8)
+ ql = (ql & np.uint8(0x0F)).reshape((n_blocks, -1))
+
+ qs = (ql | (qh << np.uint8(4))).astype(np.int8) - np.int8(16)
+
+ return (d * qs.astype(np.float32))
+
+
+class Q5_1(__Quant, qtype=GGMLQuantizationType.Q5_1):
+ @classmethod
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ max = blocks.max(axis=-1, keepdims=True)
+ min = blocks.min(axis=-1, keepdims=True)
+
+ d = (max - min) / 31
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ q = np.trunc((blocks - min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 31)
+
+ qs = q.reshape((n_blocks, 2, cls.block_size // 2))
+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))
+
+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)
+
+ d = d.astype(np.float16).view(np.uint8)
+ m = min.astype(np.float16).view(np.uint8)
+
+ return np.concatenate([d, m, qh, qs], axis=-1)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ m, rest = np.hsplit(rest, [2])
+ qh, qs = np.hsplit(rest, [4])
+
+ d = d.view(np.float16).astype(np.float32)
+ m = m.view(np.float16).astype(np.float32)
+ qh = qh.view(np.uint32)
+
+ qh = qh.reshape((n_blocks, 1)) >> np.array([i for i in range(32)], dtype=np.uint32).reshape((1, 32))
+ ql = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+ qh = (qh & np.uint32(0x01)).astype(np.uint8)
+ ql = (ql & np.uint8(0x0F)).reshape((n_blocks, -1))
+
+ qs = (ql | (qh << np.uint8(4))).astype(np.float32)
+
+ return (d * qs) + m
+
+
+class Q8_0(__Quant, qtype=GGMLQuantizationType.Q8_0):
+ @classmethod
+ # Implementation of Q8_0 with bit-exact same results as reference implementation in ggml-quants.c
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+
+ d = abs(blocks).max(axis=1, keepdims=True) / 127
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ qs = np_roundf(blocks * id)
+
+ # (n_blocks, 2)
+ d = d.astype(np.float16).view(np.uint8)
+ # (n_blocks, block_size)
+ qs = qs.astype(np.int8).view(np.uint8)
+
+ return np.concatenate([d, qs], axis=1)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ d, x = np.split(blocks, [2], axis=1)
+ d = d.view(np.float16).astype(np.float32)
+ x = x.view(np.int8).astype(np.float32)
+
+ return (x * d)
+
+
+class Q2_K(__Quant, qtype=GGMLQuantizationType.Q2_K):
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ scales, rest = np.hsplit(blocks, [QK_K // 16])
+ qs, rest = np.hsplit(rest, [QK_K // 4])
+ d, dmin = np.hsplit(rest, [2])
+
+ d = d.view(np.float16).astype(np.float32)
+ dmin = dmin.view(np.float16).astype(np.float32)
+
+ # (n_blocks, 16, 1)
+ dl = (d * (scales & 0xF).astype(np.float32)).reshape((n_blocks, QK_K // 16, 1))
+ ml = (dmin * (scales >> 4).astype(np.float32)).reshape((n_blocks, QK_K // 16, 1))
+
+ shift = np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))
+
+ qs = (qs.reshape((n_blocks, -1, 1, 32)) >> shift) & np.uint8(3)
+
+ qs = qs.reshape((n_blocks, QK_K // 16, 16)).astype(np.float32)
+
+ qs = dl * qs - ml
+
+ return qs.reshape((n_blocks, -1))
+
+
+class Q3_K(__Quant, qtype=GGMLQuantizationType.Q3_K):
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ hmask, rest = np.hsplit(blocks, [QK_K // 8])
+ qs, rest = np.hsplit(rest, [QK_K // 4])
+ scales, d = np.hsplit(rest, [12])
+
+ d = d.view(np.float16).astype(np.float32)
+
+ # The scales are packed at 6-bit each in this pattern:
+ # 0: IIIIAAAA
+ # 1: JJJJBBBB
+ # 2: KKKKCCCC
+ # 3: LLLLDDDD
+ # 4: MMMMEEEE
+ # 5: NNNNFFFF
+ # 6: OOOOGGGG
+ # 7: PPPPHHHH
+ # 8: MMIIEEAA
+ # 9: NNJJFFBB
+ # 10: OOKKGGCC
+ # 11: PPLLHHDD
+ lscales, hscales = np.hsplit(scales, [8])
+ lscales = lscales.reshape((n_blocks, 1, 8)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 2, 1))
+ lscales = lscales.reshape((n_blocks, 16))
+ hscales = hscales.reshape((n_blocks, 1, 4)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 4, 1))
+ hscales = hscales.reshape((n_blocks, 16))
+ scales = (lscales & np.uint8(0x0F)) | ((hscales & np.uint8(0x03)) << np.uint8(4))
+ scales = (scales.astype(np.int8) - np.int8(32)).astype(np.float32)
+
+ dl = (d * scales).reshape((n_blocks, 16, 1))
+
+ ql = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))
+ qh = hmask.reshape(n_blocks, -1, 1, 32) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8, 1))
+ ql = ql.reshape((n_blocks, 16, QK_K // 16)) & np.uint8(3)
+ qh = (qh.reshape((n_blocks, 16, QK_K // 16)) & np.uint8(1))
+ qh = qh ^ np.uint8(1) # strangely, the offset is zero when the bitmask is 1
+ q = (ql.astype(np.int8) - (qh << np.uint8(2)).astype(np.int8)).astype(np.float32)
+
+ return (dl * q).reshape((n_blocks, QK_K))
+
+
+class Q4_K(__Quant, qtype=GGMLQuantizationType.Q4_K):
+ K_SCALE_SIZE = 12
+
+ @staticmethod
+ def get_scale_min(scales: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+ n_blocks = scales.shape[0]
+ scales = scales.view(np.uint8)
+ ### Unpacking the following: ###
+ # 0 EEAAAAAA
+ # 1 FFBBBBBB
+ # 2 GGCCCCCC
+ # 3 HHDDDDDD
+ # 4 eeaaaaaa
+ # 5 ffbbbbbb
+ # 6 ggcccccc
+ # 7 hhdddddd
+ # 8 eeeeEEEE
+ # 9 ffffFFFF
+ # 10 ggggGGGG
+ # 11 hhhhHHHH
+ scales = scales.reshape((n_blocks, 3, 4))
+ d, m, m_d = np.split(scales, 3, axis=-2)
+
+ sc = np.concatenate([d & 0x3F, (m_d & 0x0F) | ((d >> 2) & 0x30)], axis=-1)
+ min = np.concatenate([m & 0x3F, (m_d >> 4) | ((m >> 2) & 0x30)], axis=-1)
+
+ return (sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8)))
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ dmin, rest = np.hsplit(rest, [2])
+ scales, qs = np.hsplit(rest, [cls.K_SCALE_SIZE])
+
+ d = d.view(np.float16).astype(np.float32)
+ dmin = dmin.view(np.float16).astype(np.float32)
+
+ sc, m = Q4_K.get_scale_min(scales)
+
+ d = (d * sc.astype(np.float32)).reshape((n_blocks, -1, 1))
+ dm = (dmin * m.astype(np.float32)).reshape((n_blocks, -1, 1))
+
+ qs = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+ qs = (qs & np.uint8(0x0F)).reshape((n_blocks, -1, 32)).astype(np.float32)
+
+ return (d * qs - dm).reshape((n_blocks, QK_K))
+
+
+class Q5_K(__Quant, qtype=GGMLQuantizationType.Q5_K):
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ dmin, rest = np.hsplit(rest, [2])
+ scales, rest = np.hsplit(rest, [Q4_K.K_SCALE_SIZE])
+ qh, qs = np.hsplit(rest, [QK_K // 8])
+
+ d = d.view(np.float16).astype(np.float32)
+ dmin = dmin.view(np.float16).astype(np.float32)
+
+ sc, m = Q4_K.get_scale_min(scales)
+
+ d = (d * sc.astype(np.float32)).reshape((n_blocks, -1, 1))
+ dm = (dmin * m.astype(np.float32)).reshape((n_blocks, -1, 1))
+
+ ql = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+ qh = qh.reshape((n_blocks, -1, 1, 32)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8, 1))
+ ql = (ql & np.uint8(0x0F)).reshape((n_blocks, -1, 32))
+ qh = (qh & np.uint8(0x01)).reshape((n_blocks, -1, 32))
+ q = (ql | (qh << np.uint8(4))).astype(np.float32)
+
+ return (d * q - dm).reshape((n_blocks, QK_K))
+
+
+class Q6_K(__Quant, qtype=GGMLQuantizationType.Q6_K):
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ ql, rest = np.hsplit(blocks, [QK_K // 2])
+ qh, rest = np.hsplit(rest, [QK_K // 4])
+ scales, d = np.hsplit(rest, [QK_K // 16])
+
+ scales = scales.view(np.int8).astype(np.float32)
+ d = d.view(np.float16).astype(np.float32)
+ d = (d * scales).reshape((n_blocks, QK_K // 16, 1))
+
+ ql = ql.reshape((n_blocks, -1, 1, 64)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+ ql = (ql & np.uint8(0x0F)).reshape((n_blocks, -1, 32))
+ qh = qh.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))
+ qh = (qh & np.uint8(0x03)).reshape((n_blocks, -1, 32))
+ q = (ql | (qh << np.uint8(4))).astype(np.int8) - np.int8(32)
+ q = q.reshape((n_blocks, QK_K // 16, -1)).astype(np.float32)
+
+ return (d * q).reshape((n_blocks, QK_K))
+
+
+class TQ1_0(__Quant, qtype=GGMLQuantizationType.TQ1_0):
+ @classmethod
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d = abs(blocks).max(axis=-1, keepdims=True)
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ qs = np_roundf(blocks * id)
+ qs = (qs.astype(np.int8) + np.int8(1)).astype(np.uint8)
+
+ qs0, qs1, qh = qs[..., :(32 * 5)], qs[..., (32 * 5):(48 * 5)], qs[..., (48 * 5):]
+ qs0 = qs0.reshape((n_blocks, -1, 5, 32)) * np.array([81, 27, 9, 3, 1], dtype=np.uint8).reshape((1, 1, 5, 1))
+ qs0 = np.sum(qs0, axis=-2).reshape((n_blocks, -1))
+ qs1 = qs1.reshape((n_blocks, -1, 5, 16)) * np.array([81, 27, 9, 3, 1], dtype=np.uint8).reshape((1, 1, 5, 1))
+ qs1 = np.sum(qs1, axis=-2).reshape((n_blocks, -1))
+ qh = qh.reshape((n_blocks, -1, 4, 4)) * np.array([81, 27, 9, 3], dtype=np.uint8).reshape((1, 1, 4, 1))
+ qh = np.sum(qh, axis=-2).reshape((n_blocks, -1))
+ qs = np.concatenate([qs0, qs1, qh], axis=-1)
+ qs = (qs.astype(np.uint16) * 256 + (243 - 1)) // 243
+
+ qs = qs.astype(np.uint8)
+ d = d.astype(np.float16).view(np.uint8)
+
+ return np.concatenate([qs, d], axis=-1)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ qs, rest = np.hsplit(blocks, [(QK_K - 4 * QK_K // 64) // 5])
+ qh, d = np.hsplit(rest, [QK_K // 64])
+
+ d = d.view(np.float16).astype(np.float32)
+
+ qs0, qs1 = qs[..., :32], qs[..., 32:]
+ qs0 = qs0.reshape((n_blocks, -1, 1, 32)) * np.array([1, 3, 9, 27, 81], dtype=np.uint8).reshape((1, 1, 5, 1))
+ qs0 = qs0.reshape((n_blocks, -1))
+ qs1 = qs1.reshape((n_blocks, -1, 1, 16)) * np.array([1, 3, 9, 27, 81], dtype=np.uint8).reshape((1, 1, 5, 1))
+ qs1 = qs1.reshape((n_blocks, -1))
+ qh = qh.reshape((n_blocks, -1, 1, 4)) * np.array([1, 3, 9, 27], dtype=np.uint8).reshape((1, 1, 4, 1))
+ qh = qh.reshape((n_blocks, -1))
+ qs = np.concatenate([qs0, qs1, qh], axis=-1)
+ qs = ((qs.astype(np.uint16) * 3) >> 8).astype(np.int8) - np.int8(1)
+
+ return (d * qs.astype(np.float32))
+
+
+class TQ2_0(__Quant, qtype=GGMLQuantizationType.TQ2_0):
+ @classmethod
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d = abs(blocks).max(axis=-1, keepdims=True)
+ with np.errstate(divide="ignore"):
+ id = np.where(d == 0, 0, 1 / d)
+ qs = np_roundf(blocks * id)
+ qs = (qs.astype(np.int8) + np.int8(1)).astype(np.uint8)
+
+ qs = qs.reshape((n_blocks, -1, 4, 32)) << np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))
+ qs = qs[..., 0, :] | qs[..., 1, :] | qs[..., 2, :] | qs[..., 3, :]
+ qs = qs.reshape((n_blocks, -1))
+
+ d = d.astype(np.float16).view(np.uint8)
+
+ return np.concatenate([qs, d], axis=-1)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ qs, d = np.hsplit(blocks, [QK_K // 4])
+
+ d = d.view(np.float16).astype(np.float32)
+
+ qs = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))
+ qs = (qs & 0x03).reshape((n_blocks, -1)).astype(np.int8) - np.int8(1)
+
+ return (d * qs.astype(np.float32))
+
+
+class MXFP4(__Quant, qtype=GGMLQuantizationType.MXFP4):
+ # e2m1 values (doubled)
+ # ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
+ kvalues = (0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12)
+
+ @staticmethod
+ # see ggml_e8m0_to_fp32_half in ggml-impl.h
+ def e8m0_to_fp32_half(x: np.ndarray) -> np.ndarray:
+ bits = np.where(x < 2, np.uint32(0x00200000) << np.uint32(x), np.uint32(x - 1) << np.uint32(23))
+ return bits.view(np.float32)
+
+ @classmethod
+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d = abs(blocks).max(axis=-1, keepdims=True)
+
+ with np.errstate(divide="ignore"):
+ e = np.where(d > 0, np.floor(np.log2(d)) - 2 + 127, 0).astype(np.uint8)
+
+ d = cls.e8m0_to_fp32_half(e)
+
+ kvalues = np.array(cls.kvalues, dtype=np.int8).reshape((1, 1, 16))
+
+ errs = np.abs(d.reshape((n_blocks, 1, 1)) * kvalues.astype(np.float32) - blocks.reshape((n_blocks, cls.block_size, 1)))
+ best = np.argmin(errs, axis=-1, keepdims=True)
+
+ qs = best.reshape(n_blocks, 2, cls.block_size // 2).astype(np.uint8)
+ qs = qs[:, 0] | (qs[:, 1] << np.uint8(4))
+
+ qs = qs.reshape((n_blocks, cls.block_size // 2))
+
+ return np.concatenate([e, qs], axis=-1)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ e, qs = np.hsplit(blocks, [1])
+
+ d = cls.e8m0_to_fp32_half(e)
+
+ qs = qs.reshape((n_blocks, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 2, 1))
+ qs = (qs & np.uint8(0x0F)).view(np.int8)
+
+ kvalues = np.array(cls.kvalues, dtype=np.int8).reshape(1, 1, 16)
+ qs = np.take_along_axis(kvalues, qs, axis=-1).reshape((n_blocks, cls.block_size))
+
+ return (d * qs.astype(np.float32))
+
+
+class NVFP4(__Quant, qtype=GGMLQuantizationType.NVFP4):
+ # E2M1 values doubled (kvalues_mxfp4 convention)
+ kvalues = (0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12)
+
+ @staticmethod
+ def ue4m3_to_fp32(x: np.ndarray) -> np.ndarray:
+ """Decode unsigned E4M3 (bias=7) to float, with 0.5 factor for kvalues convention."""
+ exp = (x >> 3).astype(np.int32) & 0xF
+ man = (x & 0x7).astype(np.float32)
+ raw = np.where(
+ exp == 0,
+ man * 2**-9,
+ (1.0 + man / 8.0) * (2.0 ** (exp.astype(np.float32) - 7)))
+ return np.where((x == 0) | (x == 0x7F), 0.0, raw * 0.5)
+
+ @staticmethod
+ def fp32_to_ue4m3(x: np.ndarray) -> np.ndarray:
+ """Vectorized float32 to unsigned E4M3, matching ggml_fp32_to_ue4m3 in C."""
+ x = np.clip(x, 0.0, 448.0).astype(np.float32)
+ bits = x.view(np.uint32)
+ fp32_exp = ((bits >> 23) & 0xFF).astype(np.int32) - 127
+ fp32_man = ((bits >> 20) & 0x7).astype(np.int32)
+ ue4m3_exp = fp32_exp + 7
+
+ # Subnormal
+ sub_man = np.clip((x * 512.0 + 0.5).astype(np.int32), 0, 7)
+ sub_result = np.where(sub_man >= 1, sub_man, 0).astype(np.uint8)
+
+ # Normal with rounding
+ round_bit = ((bits >> 19) & 1).astype(np.int32)
+ man = fp32_man + round_bit
+ exp = ue4m3_exp.copy()
+ overflow = man > 7
+ man = np.where(overflow, 0, man)
+ exp = np.where(overflow, exp + 1, exp)
+ normal_result = np.where(exp >= 15, np.uint8(0x7E), ((exp << 3) | man).astype(np.uint8))
+
+ return np.where(x <= 0.0, np.uint8(0),
+ np.where(ue4m3_exp <= 0, sub_result,
+ np.where(ue4m3_exp >= 15, np.uint8(0x7E), normal_result)))
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_super = blocks.shape[0]
+
+ d_bytes, qs = np.hsplit(blocks, [4])
+ d = cls.ue4m3_to_fp32(d_bytes).reshape(n_super, 4, 1) # (n_super, 4, 1)
+
+ qs = qs.reshape(n_super, 4, 8)
+ lo = (qs & np.uint8(0x0F)).view(np.int8)
+ hi = (qs >> np.uint8(4)).view(np.int8)
+ vals = np.concatenate([lo, hi], axis=-1) # (n_super, 4, 16)
+
+ kvalues = np.array(cls.kvalues, dtype=np.int8).reshape(1, 1, 16)
+ vals = np.take_along_axis(kvalues, vals, axis=-1)
+
+ return (d * vals.astype(np.float32)).reshape(n_super, 64)
+
+
+class IQ2_XXS(__Quant, qtype=GGMLQuantizationType.IQ2_XXS):
+ ksigns: bytes = (
+ b"\x00\x81\x82\x03\x84\x05\x06\x87\x88\x09\x0a\x8b\x0c\x8d\x8e\x0f"
+ b"\x90\x11\x12\x93\x14\x95\x96\x17\x18\x99\x9a\x1b\x9c\x1d\x1e\x9f"
+ b"\xa0\x21\x22\xa3\x24\xa5\xa6\x27\x28\xa9\xaa\x2b\xac\x2d\x2e\xaf"
+ b"\x30\xb1\xb2\x33\xb4\x35\x36\xb7\xb8\x39\x3a\xbb\x3c\xbd\xbe\x3f"
+ b"\xc0\x41\x42\xc3\x44\xc5\xc6\x47\x48\xc9\xca\x4b\xcc\x4d\x4e\xcf"
+ b"\x50\xd1\xd2\x53\xd4\x55\x56\xd7\xd8\x59\x5a\xdb\x5c\xdd\xde\x5f"
+ b"\x60\xe1\xe2\x63\xe4\x65\x66\xe7\xe8\x69\x6a\xeb\x6c\xed\xee\x6f"
+ b"\xf0\x71\x72\xf3\x74\xf5\xf6\x77\x78\xf9\xfa\x7b\xfc\x7d\x7e\xff"
+ )
+
+ # iq2xxs_grid, but with each byte of the original packed in 2 bits,
+ # by mapping 0x08 to 0, 0x19 to 1, and 0x2b to 2.
+ grid_shape = (256, 8)
+ grid_map = (0x08, 0x19, 0x2b)
+ grid_hex = (
+ b"00000200050008000a00110014002000220028002a0041004400500058006100"
+ b"6400800082008a00a20001010401100115014001840198010002020222028202"
+ b"010404041004210424044004420448046004810484049004a404000502050805"
+ b"200546056905800591050906100640068406a406000805080808140828084108"
+ b"440850085208880804094009020a140a01100410101021104010601084109010"
+ b"951000110811201150115a118011241245120014081420142514491480141815"
+ b"6215001616160118041810184018811800190519a019511a002002200a204420"
+ b"6120802082202921482100220222012404241024402456240025412564259026"
+ b"082820289428442a014004401040184021402440404048405640604081408440"
+ b"9040004120416141804185410142104248425642684200440844204480449944"
+ b"124524450046014804481048404845480049584961498249454a904a00500850"
+ b"1150195020508050885004514251a4519152905492540a550156545600581158"
+ b"195864584059085a046010604060686000615561186260620064056410651265"
+ b"84654268008002800a8041808280048118814081118201840484108415844084"
+ b"608400854685948509864086608602880489118a0490109024904090a1901691"
+ b"8091459200942294449451958198209902a050a085a009a100a218a450a804a9"
+ )
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, qs = np.hsplit(blocks, [2])
+
+ d = d.view(np.float16).astype(np.float32)
+
+ qs = qs.view(np.uint32).reshape(n_blocks, -1, 2)
+
+ db = d * (np.float32(0.5) + (qs[..., 1] >> 28).astype(np.float32)) * np.float32(0.25)
+ db = db.reshape((n_blocks, -1, 1, 1))
+
+ # get the sign indices and unpack the bits
+ signs = qs[..., 1].reshape((n_blocks, -1, 1)) >> np.array([0, 7, 14, 21], dtype=np.uint32).reshape((1, 1, 4))
+ ksigns = np.frombuffer(cls.ksigns, dtype=np.uint8).reshape((1, 1, 1, 128))
+ signs = (signs & np.uint32(0x7F)).reshape((n_blocks, -1, 4, 1))
+ signs = np.take_along_axis(ksigns, signs, axis=-1)
+ signs = signs.reshape((n_blocks, -1, 4, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 1, 8))
+ signs = signs & np.uint8(0x01)
+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))
+ signs = signs.reshape((n_blocks, -1, 4, 8))
+
+ assert cls.grid is not None
+ grid = np.take_along_axis(cls.grid, qs[..., 0].copy().view(np.uint8).reshape((n_blocks, -1, 1, 1)), axis=-2)
+ grid = grid.reshape((n_blocks, -1, 4, 8))
+
+ return (db * grid * signs).reshape((n_blocks, -1))
+
+
+class IQ2_XS(__Quant, qtype=GGMLQuantizationType.IQ2_XS):
+ # iq2xs_grid, but with each byte of the original packed in 2 bits,
+ # by mapping 0x08 to 0, 0x19 to 1, and 0x2b to 2.
+ grid_shape = (512, 8)
+ grid_map = (0x08, 0x19, 0x2b)
+ grid_hex = (
+ b"00000200050008000a0011001400160019002000220025002800410044004600"
+ b"49005000520055005800610064008000820085008800910094009900a0000101"
+ b"04010601090110011201150118011a0121012401400142014501480151015401"
+ b"6001680181018401900100020202050208021102140220024102440250025502"
+ b"80028a0201040404060409041004120415041804210424044004420445044804"
+ b"5104540456046004810484049004000502050505080511051405200541054405"
+ b"500561058005010604061006260640064206840600080208050808080a081108"
+ b"14082008250841084408500858088008a008aa08010904091009400981098909"
+ b"000a200a280a960aa00a01100410061009101010121015101810211024104010"
+ b"4210451048105110541060106a10811084109010001102110511081111111411"
+ b"2011411144115011801194119611011204120612101240126012001402140514"
+ b"0814111414142014411444144914501464148014011504151015401500161416"
+ b"49160118041810181218401854188618001905196619511aa91a002002200520"
+ b"08200a201120142020204120442050208020a020012104211021402148216521"
+ b"002222228022a82201240424102429244024002541255225992501261a26a626"
+ b"002808280a28202855288828a22868299029082a202a822a882a8a2a01400440"
+ b"0640094010401240154018402140244040404240454048404a40514054406040"
+ b"6540814084409040004102410541084111411441204141414441504180418541"
+ b"a241014204421042124229424042004402440544084411441444194420444144"
+ b"4444504480449444014504451045244540459a4500460a464446504601480448"
+ b"1048404845485448624800491149444950496949044a00500250055008501150"
+ b"145020502850415044505050805001510451105115514051425100524452aa52"
+ b"0154045410542154405460548154a154005508558055885521566856a1560058"
+ b"14584158505899581a5940594259855a0160046010604060546062608660a960"
+ b"006124624a62926200641664106540654565a46501686a682569066a546a626a"
+ b"00800280058008801180148020802a8041804480508080808280a880aa800181"
+ b"0481068110814081518159810082208280828282a082a8820184048410841284"
+ b"158440846084898400854485a58518866a860088088825885a8880888288a888"
+ b"0689228a808a888a968aa88a0190049010904090569084900091229164915692"
+ b"89920094059444945094589429959095929541965198a6984999159a609a00a0"
+ b"02a008a00aa020a02aa0a0a051a159a1a6a100a202a208a22aa280a2a0a240a4"
+ b"95a465a698a60aa820a822a828a8a0a8a8a804a984a986a928aa2aaa91aaaaaa"
+ )
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ qs, scales = np.hsplit(rest, [2 * QK_K // 8])
+
+ d = d.view(np.float16).astype(np.float32)
+ qs = qs.view(np.uint16)
+
+ scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))
+ scales = (scales & 0x0F).reshape((n_blocks, -1))
+ db = d * (np.float32(0.5) + scales) * np.float32(0.25)
+ db = db.reshape((n_blocks, -1, 1, 1))
+
+ # get the sign indices and unpack the bits
+ signs = np.frombuffer(IQ2_XXS.ksigns, dtype=np.uint8).reshape(1, 1, 128)
+ signs = np.take_along_axis(signs, (qs >> 9).reshape((n_blocks, -1, 1)), axis=-1)
+ signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8))
+ signs = signs & np.uint8(0x01)
+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))
+ signs = signs.reshape((n_blocks, -1, 2, 8))
+
+ assert cls.grid is not None
+ grid = np.take_along_axis(cls.grid, (qs & np.uint16(511)).reshape((n_blocks, -1, 1, 1)), axis=-2)
+ grid = grid.reshape((n_blocks, -1, 2, 8))
+
+ return (db * grid * signs).reshape((n_blocks, -1))
+
+
+class IQ2_S(__Quant, qtype=GGMLQuantizationType.IQ2_S):
+ # iq2s_grid, but with each byte of the original packed in 2 bits,
+ # by mapping 0x08 to 0, 0x19 to 1, and 0x2b to 2.
+ grid_shape = (1024, 8)
+ grid_map = (0x08, 0x19, 0x2b)
+ grid_hex = (
+ b"00000200050008000a0011001400160019002000220025002800410044004600"
+ b"490050005200550058006100640066006900800082008500880091009400a000"
+ b"a500aa0001010401060109011001120115011801210124014001420145014801"
+ b"510154015601590160016501680181018401900192019501a101a40100020202"
+ b"050208021102140220022a02410244024602490250025502800285028a029402"
+ b"a202010404040604090410041204150418042104240426042904400442044504"
+ b"48044a0451045404560459046004620465048104840486048904900495049804"
+ b"a104a40400050205050508050a05110514051605190520052505280541054405"
+ b"46054905500552055505580561056405800582058505880591059405a0050106"
+ b"0406060609061006150640064506480651065406600681068406900600080208"
+ b"050808081108140816081908200825082a084108440846084908500852085508"
+ b"580861086408800885089408aa08010904091009120915091809210940094509"
+ b"480951095409600981099009000a110a140a220a280a2a0a500a990a01100410"
+ b"0610091010101210151018102110241026104010421045104810511054105610"
+ b"59106010621065106810811084108610901095109810a110a410001102110511"
+ b"08110a1111111411161119112011221125112811411144114611491150115211"
+ b"5511581161116411801182118511881191119411011204120912101215122112"
+ b"2412401245125112541281128412901200140214051408141114141416141914"
+ b"2014251428144114441446144914501452145514581461146414801482148514"
+ b"881491149414a014011504150615091510151215151518152115241540154215"
+ b"4515481551155415601581158415901500160516081611161416201641164416"
+ b"50168016aa160118041806180918101815181818211840184218451848185118"
+ b"541860188118841800190219051908191119141920194119441950196919a219"
+ b"041a101a401a561a00200220052008201120142016201920202025202a204120"
+ b"4420502052205520642080208a209420aa200121042110211221152121214021"
+ b"4221452151215421602181218421902100220a22222228222a22442250228822"
+ b"8a22a82201240424062409241024152418242124242440244224452448245124"
+ b"5424602481248424902400250525082511251425202541254425502566258025"
+ b"0126042610264026592600280528112814284128442850288a28aa2801290429"
+ b"102995290a2a222a642a882a8a2a014004400640094010401240154018401a40"
+ b"21402440264040404240454048404a4051405440564059406040624065408140"
+ b"8440904095409840a140a4400041024105410841114114411641194120412241"
+ b"2541414144414641494150415241554158416141644180418241854188419141"
+ b"9441a04101420442104212421542184224424042454248425142544260428142"
+ b"844200440244054408440a441144144416441944204422442544284441444444"
+ b"46444944504452445544584461446444804482448544884491449444a0440145"
+ b"0445064509451045124515451845214524454045424545454845514554456045"
+ b"6a4581458445904500460246054608461146144620464146444650468046a546"
+ b"0148044809481048124815481848214824484048424845484848514854486048"
+ b"84489048004902490549084911491449204941494449504980499649014a044a"
+ b"104a404a00500250055008501150145016501950205022502550285041504450"
+ b"4650495050505250555058506150645080508250855088509150945001510451"
+ b"0651095110511251155118512151245140514251455148515151545160518151"
+ b"8451905100520552085211521452205241524452505269528052015404540654"
+ b"0954105412541554185421542454405442544554485451545454605481548454"
+ b"9054005502550555085511551455205541554455505580550156045610562656"
+ b"405600580258055808581158145820584158445850585a588058015904591059"
+ b"4059005a195a855aa85a01600460066010601260156018602160246040604560"
+ b"4860516054606060846090600061026105610861116114612061416144615061"
+ b"806199610462106240625662a162006405640864116414642064416444645064"
+ b"806401650465106540654a656865926500669466016804681068656898680069"
+ b"2a69426aa16a0080028005800880118014801980208025804180448050805280"
+ b"5580588061808080858091809480018104810981108112811581188121812481"
+ b"408142814581488151815481818184819081a981008205820a82118214824182"
+ b"4482508201840484068409841084128415841884218440844284458448845184"
+ b"5484608481848484908400850285058508851185148520854185448550858085"
+ b"8a85018604861086298640860088058811881488418844885088a28801890489"
+ b"40896589228a588a5a8a828aa28a019004900990109012901590189024904090"
+ b"4290459048905190549060908190849090900091059111911491419144915091"
+ b"5a910192049210924092a6920094029405940894119414942094419444945094"
+ b"8094969401950495109540959895a19500964696649601980498109826984098"
+ b"a998009949995299909a00a005a00aa014a022a02aa041a044a050a0a2a0aaa0"
+ b"40a165a102a20aa222a228a22aa282a288a28aa2a8a201a404a410a440a489a4"
+ b"a4a400a519a551a60aa828a8a2a854a986a908aa0aaa20aa22aa28aa88aaaaaa"
+ )
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ qs, rest = np.hsplit(rest, [QK_K // 8])
+ signs, rest = np.hsplit(rest, [QK_K // 8])
+ qh, scales = np.hsplit(rest, [QK_K // 32])
+
+ d = d.view(np.float16).astype(np.float32)
+
+ scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))
+ scales = (scales & 0x0F).reshape((n_blocks, -1))
+ db = d * (np.float32(0.5) + scales) * np.float32(0.25)
+ db = db.reshape((n_blocks, -1, 1, 1))
+
+ # unpack the sign bits
+ signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8))
+ signs = signs & np.uint8(0x01)
+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))
+ signs = signs.reshape((n_blocks, -1, 2, 8))
+
+ qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4))
+ qs = qs.astype(np.uint16) | ((qh & 0x03).astype(np.uint16) << 8).reshape((n_blocks, -1))
+
+ assert cls.grid is not None
+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)
+ grid = grid.reshape((n_blocks, -1, 2, 8))
+
+ return (db * grid * signs).reshape((n_blocks, -1))
+
+
+class IQ3_XXS(__Quant, qtype=GGMLQuantizationType.IQ3_XXS):
+ grid_shape = (256, 4)
+ grid_map = (0x04, 0x0c, 0x14, 0x1c, 0x24, 0x2c, 0x34, 0x3e)
+ grid_hex = (
+ b"0000020004001100130017002000220031004200730075000101030110011201"
+ b"2101250130013201410154017001000202020402110220022202310233023702"
+ b"5102570275020103070310031203250370031304370444045704730475040105"
+ b"0705320552053506640610071407160743076107011003101010121021102310"
+ b"3010321034104710501000110211111120112211011203121012121221123012"
+ b"7212001302132013311346136613011405145014201524154615711505162217"
+ b"4017002002201120132020202220262031204220012103210521102112212121"
+ b"3021632167217021002202221122172220222222372240225522012310231423"
+ b"7023742335245324032527254125742501270327162745270130103012302130"
+ b"2330503065307230003102312031313144314631013203321032253252327232"
+ b"1133333330344734723400350635223555351436363663363337603704401740"
+ b"3540374053405740744120423742404260426642074345430444514464442545"
+ b"4345704505471047124730471250415070500051065126515551145232527252"
+ b"0253535310542354275472540255315550562457425724604460466064602161"
+ b"6161176264623063366344640565526533660367216703700570077010703270"
+ b"5270267140711272457252720073157333736073217441740075027524753076"
+ )
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ qs, scales = np.hsplit(rest, [QK_K // 4])
+
+ d = d.view(np.float16).astype(np.float32)
+ scales = scales.view(np.uint32)
+
+ db = d * (np.float32(0.5) + (scales >> 28).astype(np.float32)) * np.float32(0.5)
+ db = db.reshape((n_blocks, -1, 1, 1))
+
+ # get the sign indices and unpack the bits
+ signs = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 7, 14, 21], dtype=np.uint32).reshape((1, 1, 4))
+ ksigns = np.frombuffer(IQ2_XXS.ksigns, dtype=np.uint8).reshape((1, 1, 1, 128))
+ signs = (signs & np.uint32(0x7F)).reshape((n_blocks, -1, 4, 1))
+ signs = np.take_along_axis(ksigns, signs, axis=-1)
+ signs = signs.reshape((n_blocks, -1, 4, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 1, 8))
+ signs = signs & np.uint8(0x01)
+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))
+ signs = signs.reshape((n_blocks, -1, 4, 8))
+
+ assert cls.grid is not None
+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)
+ grid = grid.reshape((n_blocks, -1, 4, 8))
+
+ return (db * grid * signs).reshape((n_blocks, -1))
+
+
+class IQ3_S(__Quant, qtype=GGMLQuantizationType.IQ3_S):
+ grid_shape = (512, 4)
+ grid_map = (0x01, 0x03, 0x05, 0x07, 0x09, 0x0b, 0x0d, 0x0f)
+ grid_hex = (
+ b"0000010002000500070010001100120014001600200021002500330040004200"
+ b"4500470051005300600062007100740077000001010102010401100111011501"
+ b"2001230127013101350144016101650172010002010205020702100213021602"
+ b"2102250230023402420245024702510253027002730203031103150320032203"
+ b"3103330336034403500352036703710375030004130417042104240432044004"
+ b"4304510470040205040520052205260533054105450547056605730506061106"
+ b"1306310652067106000702070407200722072607330750075407001001100210"
+ b"0410101011101310151017102010221031103410361054105610611072100011"
+ b"0111031106111011141121113011331141115011521170117611001212121512"
+ b"1712201224123212401243125512601272120113041307131013131321132713"
+ b"3013341341136213701303140514121414143114331442144614501454140115"
+ b"1015131521153015321551152016241627164416461601170317101712172117"
+ b"3517411762177017002001200320052007201020122014201620212023202720"
+ b"3020322041204320452050205220672070207320752000210221102113211721"
+ b"2221252131213421422151210122042207222122232230223722412253225722"
+ b"7122742200230223052311232223242331233323422350236623012407242024"
+ b"2324322435244124722475240425112522253725402553257025002602260726"
+ b"2126552661260527112726273027432750270230113013301530173022303130"
+ b"3330353042304430473051306330713001310331053114312131233140316031"
+ b"7231763100321232203232323432503201331033143321332333273330334133"
+ b"4333473355337333033411341634223431345234603464340135103512352535"
+ b"3235443556357335163641360137033720372237353700400440124020402440"
+ b"2740324041405040704002410741114113412241304135414341514155410142"
+ b"0342104215422142334240425742624270420443114313432043224331433543"
+ b"0044024424443744404471440545074521456245134634466046104715473047"
+ b"4347514702501050145022504050445047505250665074500151035105511251"
+ b"2151325172510052115223523052365253520253075310532753445351536553"
+ b"7353015404542054325446541255265551555355425602570457225711601360"
+ b"1560316033606060006120612761646112623462426255626262706200631463"
+ b"2163406325644364626400650365346560650566406611671367007004700770"
+ b"2070227036704070547062700271117124714371457101720472107216722172"
+ b"3072517202733273357353730174057413742074507422754275027631760077"
+ )
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ qs, rest = np.hsplit(rest, [QK_K // 4])
+ qh, rest = np.hsplit(rest, [QK_K // 32])
+ signs, scales = np.hsplit(rest, [QK_K // 8])
+
+ d = d.view(np.float16).astype(np.float32)
+
+ scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))
+ scales = (scales & 0x0F).reshape((n_blocks, -1))
+ db = d * (1 + 2 * scales)
+ db = db.reshape((n_blocks, -1, 1, 1))
+
+ # unpack the sign bits
+ signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8))
+ signs = signs & np.uint8(0x01)
+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))
+ signs = signs.reshape((n_blocks, -1, 4, 8))
+
+ qh = qh.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8)], dtype=np.uint8)
+ qh = (qh & 0x01).astype(np.uint16).reshape((n_blocks, -1))
+ qs = qs.astype(np.uint16) | (qh << 8)
+
+ assert cls.grid is not None
+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)
+ grid = grid.reshape((n_blocks, -1, 4, 8))
+
+ return (db * grid * signs).reshape((n_blocks, -1))
+
+
+class IQ1_S(__Quant, qtype=GGMLQuantizationType.IQ1_S):
+ # iq1s_grid, with each byte packed into 2 bits
+ # -1, 0, 1 <=> 0, 1, 2
+ grid_shape = (2048, 8)
+ grid_map = (-1, 0, 1)
+ grid_hex = (
+ b"00000200050008000a00110015002000220028002a0045005100540056006500"
+ b"8000820088008a009500a000a200a800aa000401050111011401160119011a01"
+ b"2501410146014901520155015a0161016401660168018501910194019601a501"
+ b"0002020208020a0215022002220228022a024502510259026402690280028202"
+ b"88028a02910295029902a002a202a802aa021104140416042504410449045504"
+ b"5a046404650491049904a5040105040505050605150518051a05290540054505"
+ b"4a0550055105540555055605590560056205650568056a058105910595059805"
+ b"9a05a105a405a505a605a9051406190641064406500652065506580660066106"
+ b"6606690685069106940699060008020808080a0815082008220828082a084508"
+ b"5108560865088008820888088a089508a008a208a808aa080509110914091909"
+ b"2409250941095009510955096109640969099109940996099909a509000a020a"
+ b"080a0a0a150a200a220a280a2a0a450a510a590a610a650a800a820a850a880a"
+ b"8a0a950aa00aa20aa80aaa0a1010111014101910241025104110441050105510"
+ b"58106110641065106910911094109610a110a510011104110611091110111211"
+ b"1511181121112411291145114a11501151115211541155115611591160116511"
+ b"841192119511a111a41111121412161225124012461249125212551258125a12"
+ b"641266128512911294129612a512011406140914141415141814191421142614"
+ b"41144514461448144a1451145414551456145914621465146814841489149014"
+ b"94149514981499149a14a114a414a514a914021505150a151115141515151615"
+ b"191520152215251528152a154115441545154615511552155415551556155915"
+ b"5a1561156415651566156915801582158415851588158a159015911594159515"
+ b"961599159a15a015a215a51501160416051606161516161618161a1621162616"
+ b"401642164416451648164a165116551656165816591661166416651668166916"
+ b"6a1686168a1692169516a416a916111816182518411844184618491850185518"
+ b"58185a1860186118641866186918851891189418a5181019121915191a192119"
+ b"25194219441945194819511954195519561959195a19601965196a1989199119"
+ b"921995199819a119a619a919091a161a241a261a441a461a491a501a521a551a"
+ b"581a611a661a691a851a911a961a9a1a0020022008200a201520202022202520"
+ b"28202a20452051205920612065208020822088208a209520a020a220a520a820"
+ b"aa2005211121142119212521422144214921552158215a216121642165216621"
+ b"8521902196219921a521012208220a22112215222022222228222a2245225122"
+ b"562259226522812288228a2291229522a022a222a822aa220524142416241924"
+ b"252444244524462449245224552458245a2466248524912494249924a124a524"
+ b"0925152521252925402545254825512554255525592562256525682589259025"
+ b"9425952598259a25a125a425a625a92505261026122619262526412649265526"
+ b"6026612669268426862690269a260028022808280a2815282028222828282a28"
+ b"45285128542865288028822888288a28a028a228a828aa280929112914291929"
+ b"2529462949295229552961296429662969298529902996299929a429a529002a"
+ b"022a082a0a2a202a222a282a2a2a452a512a562a592a652a802a822a882a8a2a"
+ b"952aa02aa22aa82aaa2a054011401640254049405240554058405a4061406440"
+ b"664094409940a140a6400041014104410641094112411541164118411a412141"
+ b"26412941454148414a41514154415541564159415a41654168416a4181418441"
+ b"8641904192419541a041a141a241054211421442164225424142524255425a42"
+ b"6442694289429442a5420144154419442944454448444a445144544455445644"
+ b"61446244654468446a44814486448944904492449544a044a144a94401450245"
+ b"05450a4511451445154516451945204525452a45414544454545464549455045"
+ b"5145544555455645584559456145644565456645694582458445854588459145"
+ b"94459545964599459a45a545a845aa450146054609461446154618461a462146"
+ b"2446294640464246454648465046514652465546564659466246654668468146"
+ b"85468a4694469546a146a446a6460548114815481a4825484248494850485548"
+ b"5848614864486648694885489148944896489948a5480149054906490a491049"
+ b"144915491849214924492649404945494a495149524954495549564959496049"
+ b"6249654966496a49864989499249954996499849a149a449a649a949164a444a"
+ b"464a494a554a584a5a4a644a694a944aa54a0150045005500650095012501550"
+ b"1a50215024502950405045504850515054505550565059506550685086508950"
+ b"95509850a050a150a650a9500551085109510a51115114511551165118511951"
+ b"20512551265128512a5141514451455146514951505151515251545155515651"
+ b"585159515a51615164516551665169518251855191519451955196519951a051"
+ b"a551aa5101520652125215521a5221522452425245524a525152545255525652"
+ b"595262526552855290529252955299529a52a452045405541154145415541654"
+ b"185419542154255428542a54415444544554465449544a545054515454545554"
+ b"5654585459545a54615462546454655466546954805488548a54915494549554"
+ b"96549954a154a454a554aa540155025504550555065509551055115512551455"
+ b"1555165519551a55215524552555265529554055415542554455455546554855"
+ b"4955505551555255545555555655585559555a55605561556455655566556855"
+ b"69556a5581558455855589558a559055915594559555965598559955a155a455"
+ b"a555a655a9550056015602560456065608560956115614561556185619562056"
+ b"2156225624562556265628562956415645564656485649564a56505651565256"
+ b"545655565656585659565a566156645665566956825685568656885689568a56"
+ b"915695569a56a256a556a656a856a95604580558065809581058155818582158"
+ b"2a58455848584a58515854585558565858585958605862586458655882588958"
+ b"9058925895589858a158a9580159025905590a59115914591559165919592559"
+ b"41594459455946594959505951595259545955595659585959595a5961596459"
+ b"655966596959815985598959915994599559965998599959a559045a085a155a"
+ b"1a5a205a255a265a295a455a485a495a515a555a565a585a595a625a655a685a"
+ b"6a5a815a8a5a925a955a965a985a9a5aa15a0560146016601960256044605060"
+ b"5560566058605a60616064606660696081609660a56001610461066109611261"
+ b"15612161226126612961456149615161556156615961656166616a6184618a61"
+ b"92619561a161a661a96111621662196240624162466255625662586260628562"
+ b"91629662a56211641264156416641a6421642664296440644264456448644a64"
+ b"516454645564566459645a646064626465648464856489649064926494649564"
+ b"966498649a64a164a464a964056508650a651165156516651965446545654665"
+ b"496550655165546555655665596561656465656566656965866589658a659165"
+ b"9565966599659a65a265a565a665a86502660966156620662666286629664066"
+ b"456648664a66516654665566566658665a666066656668668066826685668a66"
+ b"9466966698669966a066a466a666aa661668196825684168526855685a686168"
+ b"6968856891689868a66801690469106915692169246926692969406941694569"
+ b"4669486951695469556956695969606965696a69826984698a699569a169a469"
+ b"a569a969116a166a186a416a446a496a506a556a586a5a6a646a656a696a866a"
+ b"946a986a9a6aa66a0080028008800a802080228028802a804580508051805480"
+ b"5680598065808080828088808a809580a080a280a880aa800581118114811681"
+ b"1981258141814481498150815281558156815881598164816681698185818981"
+ b"948196819981a5810082028208820a8215822082228228822a82518254825982"
+ b"65828082828288828a829582a082a282a882aa82148419844184448451845584"
+ b"5a846184648469849484998401850985128515851a8526852985408541854585"
+ b"4885518554855585568559855a856585668568856a8581858485868589859085"
+ b"928595859885a68511861686198625864186448649864a865086558659865a86"
+ b"618666866a86858691869a86a4860088028808880a8815882088228828882a88"
+ b"41884588518854885988658869888088828888888a889588a088a288a888aa88"
+ b"05890689118914891689258941894489468949895089528955895a8961896489"
+ b"858996899989a589008a028a088a0a8a158a208a228a288a2a8a458a518a548a"
+ b"568a808a828a888a8a8a958aa08aa28aa88aaa8a059011901690189019902590"
+ b"419046904990559058905a9069906a9085909190949096909990a59001910491"
+ b"069109911091159118911a912191249126912991409145915091519154915591"
+ b"569159916291659184918691929195919891a191a491a691a991059211921492"
+ b"19922592449246924992509252925592589266926992859294929692a9920194"
+ b"04940694109415941894269440944a9451945494559456945894599460946194"
+ b"62946594849486949294949495949894a194a9940095059508950a9510951195"
+ b"14951595169519952195259529952a9541954495459546954995509551955295"
+ b"549555955695589559955a956195649565956695699581958595889591959295"
+ b"94959595969599959a95a095a295a595a895aa95019604961096159619962096"
+ b"2696299645964896499651965296559656965996659668968296849689968a96"
+ b"929694969596a496a696a9960598169819982598419846985098529855985698"
+ b"5a98649865988598919896989998a59804990699099910991299159918991a99"
+ b"209921992499269940994299459948994a995199549955995699599962996599"
+ b"66996a99819984999099929995999a99a199a699059a159a259a449a469a499a"
+ b"509a559a589a619a859a919a949a959a969a00a002a008a00aa015a020a022a0"
+ b"28a02aa045a051a054a056a059a080a082a088a08aa095a0a0a0a2a0a8a0aaa0"
+ b"05a109a111a114a116a119a11aa146a149a151a155a158a15aa161a164a185a1"
+ b"90a192a196a199a102a208a20aa210a219a222a228a22aa245a251a256a259a2"
+ b"65a280a282a288a28aa295a2a0a2a2a2a8a2aaa219a425a441a444a450a454a4"
+ b"55a458a45aa461a465a466a468a469a485a406a509a510a512a515a518a526a5"
+ b"29a542a545a551a554a555a556a559a565a56aa581a584a585a586a589a592a5"
+ b"95a598a505a611a616a61aa621a625a644a646a64aa652a655a656a658a660a6"
+ b"62a686a690a695a696a699a6a1a6a4a6a6a600a802a808a80aa820a822a828a8"
+ b"2aa851a854a856a859a880a882a888a88aa895a8a0a8a2a8a8a8aaa805a914a9"
+ b"19a921a925a941a950a955a95aa961a966a969a990a996a900aa02aa08aa0aaa"
+ b"20aa22aa28aa2aaa51aa54aa56aa80aa82aa88aa8aaa95aaa0aaa2aaa8aaaaaa"
+ )
+
+ delta = np.float32(0.125)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ qs, qh = np.hsplit(rest, [QK_K // 8])
+
+ d = d.view(np.float16).astype(np.float32)
+ qh = qh.view(np.uint16)
+
+ dl = d * (2 * ((qh >> 12) & 7) + 1)
+ dl = dl.reshape((n_blocks, -1, 1, 1))
+ delta = np.where((qh & np.uint16(0x8000)) == 0, cls.delta, -cls.delta)
+ delta = delta.reshape((n_blocks, -1, 1, 1))
+
+ qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 3, 6, 9], dtype=np.uint16).reshape((1, 1, 4))
+ qs = qs.astype(np.uint16) | ((qh & 7) << 8).reshape((n_blocks, -1))
+
+ assert cls.grid is not None
+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)
+ grid = grid.reshape((n_blocks, -1, 4, 8))
+
+ return (dl * (grid + delta)).reshape((n_blocks, -1))
+
+
+class IQ1_M(__Quant, qtype=GGMLQuantizationType.IQ1_M):
+ grid_shape = IQ1_S.grid_shape
+ grid_map = IQ1_S.grid_map
+ grid_hex = IQ1_S.grid_hex
+
+ delta = IQ1_S.delta
+
+ # Okay *this* type is weird. It's the only one which stores the f16 scales in multiple parts.
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ qs, rest = np.hsplit(blocks, [QK_K // 8])
+ qh, scales = np.hsplit(rest, [QK_K // 16])
+
+ # The f16 scale is packed across multiple bytes
+ scales = scales.view(np.uint16)
+ d = (scales.reshape((n_blocks, 4)) & np.uint16(0xF000)) >> np.array([12, 8, 4, 0], dtype=np.uint16).reshape((1, 4))
+ d = d[..., 0] | d[..., 1] | d[..., 2] | d[..., 3]
+ d = d.view(np.float16).astype(np.float32).reshape((n_blocks, 1))
+
+ scales = scales.reshape(n_blocks, -1, 1) >> np.array([0, 3, 6, 9], dtype=np.uint16).reshape((1, 1, 4))
+ scales = (scales & 0x07).reshape((n_blocks, -1))
+ dl = d * (2 * scales + 1)
+ dl = dl.reshape((n_blocks, -1, 2, 1, 1))
+
+ qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))
+ qs = qs.astype(np.uint16) | ((qh & 0x07).astype(np.uint16) << 8).reshape((n_blocks, -1))
+
+ delta = np.where(qh & 0x08 == 0, cls.delta, -cls.delta)
+ delta = delta.reshape((n_blocks, -1, 2, 2, 1))
+
+ assert cls.grid is not None
+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)
+ grid = grid.reshape((n_blocks, -1, 2, 2, 8))
+
+ return (dl * (grid + delta)).reshape((n_blocks, -1))
+
+
+class IQ4_NL(__Quant, qtype=GGMLQuantizationType.IQ4_NL):
+ kvalues = (-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113)
+
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, qs = np.hsplit(blocks, [2])
+
+ d = d.view(np.float16).astype(np.float32)
+
+ qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+
+ qs = (qs & np.uint8(0x0F)).reshape((n_blocks, -1, 1))
+
+ kvalues = np.array(cls.kvalues, dtype=np.int8).reshape(1, 1, 16)
+ qs = np.take_along_axis(kvalues, qs, axis=-1).astype(np.float32).reshape((n_blocks, -1))
+
+ return (d * qs)
+
+
+class IQ4_XS(__Quant, qtype=GGMLQuantizationType.IQ4_XS):
+ @classmethod
+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+ n_blocks = blocks.shape[0]
+
+ d, rest = np.hsplit(blocks, [2])
+ scales_h, rest = np.hsplit(rest, [2])
+ scales_l, qs = np.hsplit(rest, [QK_K // 64])
+
+ d = d.view(np.float16).astype(np.float32)
+ scales_h = scales_h.view(np.uint16)
+
+ scales_l = scales_l.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))
+ scales_h = scales_h.reshape((n_blocks, 1, -1)) >> np.array([2 * i for i in range(QK_K // 32)], dtype=np.uint16).reshape((1, -1, 1))
+ scales_l = scales_l.reshape((n_blocks, -1)) & np.uint8(0x0F)
+ scales_h = scales_h.reshape((n_blocks, -1)).astype(np.uint8) & np.uint8(0x03)
+
+ scales = (scales_l | (scales_h << np.uint8(4))).astype(np.int8) - np.int8(32)
+ dl = (d * scales.astype(np.float32)).reshape((n_blocks, -1, 1))
+
+ qs = qs.reshape((n_blocks, -1, 1, 16)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))
+ qs = qs.reshape((n_blocks, -1, 32, 1)) & np.uint8(0x0F)
+
+ kvalues = np.array(IQ4_NL.kvalues, dtype=np.int8).reshape((1, 1, 1, -1))
+ qs = np.take_along_axis(kvalues, qs, axis=-1).astype(np.float32).reshape((n_blocks, -1, 32))
+
+ return (dl * qs).reshape((n_blocks, -1))
diff --git a/tests/kernels/test_ggml.py b/tests/kernels/test_ggml.py
new file mode 100644
index 00000000..ea6a5d68
--- /dev/null
+++ b/tests/kernels/test_ggml.py
@@ -0,0 +1,84 @@
+"""Packed GGUF ops against gguf-py's CPU dequantization and PyTorch math."""
+
+import numpy as np
+import pytest
+import torch
+import torch.nn.functional as F
+
+from freetoken.kernel import ggml
+from freetoken.third_party.gguf import GGMLQuantizationType as QType
+from freetoken.third_party.gguf import quants
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a GPU")
+DTYPES = [torch.float32, torch.bfloat16, torch.float16]
+
+
+def _q4_weights(shape, seed=690):
+ values = np.random.default_rng(seed).normal(0, 0.1, shape).astype(np.float32)
+ packed = quants.quantize(values, QType.Q4_0)
+ reference = torch.from_numpy(quants.dequantize(packed, QType.Q4_0))
+ # ggml's last MMQ tile may read an extra partial row.
+ storage = torch.zeros(*packed.shape[:-2], packed.shape[-2] + 1, packed.shape[-1], dtype=torch.uint8, device="cuda")
+ storage[..., :-1, :].copy_(torch.from_numpy(packed))
+ return storage[..., :-1, :], reference
+
+
+def _activation(shape, dtype, seed=19):
+ rng = np.random.default_rng(seed)
+ half = rng.integers(-127, 128, (*shape[:-1], shape[-1] // 32, 16))
+ half[..., 0] = 127
+ # Exactly representable q8_1 inputs with zero block sums isolate weight math.
+ values = np.concatenate([half, -half], axis=-1).reshape(shape).astype(np.float32) / 64
+ return torch.from_numpy(values).to("cuda", dtype)
+
+
[email protected]("dtype", DTYPES)
+def test_q4_dequantize_and_rows(dtype):
+ w, ref = _q4_weights((64, 256))
+ actual = ggml.dequantize(w, int(QType.Q4_0), 256, dtype)
+ torch.testing.assert_close(actual.cpu(), ref.to(dtype), atol=0, rtol=0)
+ ids = torch.tensor([63, 0, 17, 17], dtype=torch.int32, device="cuda")
+ rows = ggml.get_rows(w, int(QType.Q4_0), 256, ids, dtype)
+ torch.testing.assert_close(rows.cpu(), ref[ids.cpu().long()].to(dtype), atol=0, rtol=0)
+
+
[email protected]("dtype", DTYPES)
[email protected]("batch", [1, 17])
+def test_q4_matrix_matches_cpu(dtype, batch):
+ w, ref = _q4_weights((64, 256))
+ x = _activation((batch, 256), dtype)
+ actual = ggml.mul_mat(w, int(QType.Q4_0), x)
+ expected = F.linear(x.cpu().float(), ref).to(dtype)
+ torch.testing.assert_close(actual.cpu(), expected, atol=2e-4, rtol=1e-3)
+
+
[email protected]("dtype", DTYPES)
[email protected]("batch", [1, 17])
+def test_q4_moe_routes_match_cpu(dtype, batch):
+ w, ref = _q4_weights((4, 64, 256))
+ x = _activation((batch, 256), dtype)
+ ids = torch.tensor([[i % 4, (i + 1) % 4] for i in range(batch)], dtype=torch.int32, device="cuda")
+ actual = ggml.mul_mat_id(w, int(QType.Q4_0), x, ids)
+ expected = torch.einsum("tk, tjok -> tjo", x.cpu().float(), ref[ids.cpu().long()]).to(dtype)
+ torch.testing.assert_close(actual.cpu(), expected, atol=2e-4, rtol=1e-3)
+
+
[email protected]("dtype", DTYPES)
+def test_q4_shared_q8_matches_internal_quantization(dtype):
+ w, _ = _q4_weights((64, 256))
+ x = _activation((1, 256), dtype)
+ expected = ggml.mul_mat(w, int(QType.Q4_0), x)
+ actual = ggml.mul_mat(w, int(QType.Q4_0), x, xq=ggml.quantize_q8_1(x))
+ torch.testing.assert_close(actual, expected, atol=0, rtol=0)
+
+
+def test_q4_nondefault_stream():
+ w, ref = _q4_weights((64, 256))
+ torch.cuda.synchronize()
+ stream = torch.cuda.Stream()
+ with torch.cuda.stream(stream):
+ x = _activation((1, 256), torch.float32)
+ actual = ggml.mul_mat(w, int(QType.Q4_0), x)
+ copied = actual.clone()
+ stream.synchronize()
+ torch.testing.assert_close(copied.cpu(), F.linear(x.cpu(), ref), atol=2e-4, rtol=1e-3)
diff --git a/tests/models/deepseek_v41/harness.py b/tests/models/deepseek_v41/harness.py
index 98b19e2d..83c0ec61 100644
--- a/tests/models/deepseek_v41/harness.py
+++ b/tests/models/deepseek_v41/harness.py
@@ -62,7 +62,7 @@ class TinyEngine:
from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache
self.banks = load_expert_banks(
- checkpoint, mc, method=method, device=self.device, dtype=torch.bfloat16, parallel=False,
+ checkpoint, mc, method=method, device=self.device, parallel=False,
)
self.expert_cache = OffloadMoeCache(
num_layers=mc.num_moe_layers, num_experts=mc.num_experts,
@@ -76,7 +76,7 @@ class TinyEngine:
from freetoken.moe.expert_banks import attach_resident_banks
attach_resident_banks(list(iter_moe_layers(self.model)), load_expert_banks(
- checkpoint, mc, method=method, device=self.device, dtype=torch.bfloat16, parallel=False, resident=True,
+ checkpoint, mc, method=method, device=self.device, parallel=False, resident=True,
))
for layer in self.model.engram_layers():
layer.attach_table(engram_table if engram_table is not None else ZeroEngramTable(layer.width, self.device))
diff --git a/tests/moe/test_cpu_moe_q4_0.py b/tests/moe/test_cpu_moe_q4_0.py
index 7babb5d4..9452f95c 100644
--- a/tests/moe/test_cpu_moe_q4_0.py
+++ b/tests/moe/test_cpu_moe_q4_0.py
@@ -3,7 +3,7 @@ the gemma4 GGUF checkpoint).
The CPU W4A16 GEMV (``q4_0_dot`` in csrc/cpu_moe/kernels/q4_0.cpp) reads the *same*
packed Q4_0 banks the GPU offload path streams and dequantizes weights inside the
-K-loop. We check it against the reference dequant (models/gguf/dequant.py) + the
+K-loop. We check it against the reference dequant (gguf-py's numpy dequantize) + the
production bf16 GPU decode kernel on byte-identical banks: both are W4A16, so the
only spread is weight bf16-rounding + reduction order -> tight relative tolerance.
@@ -39,11 +39,11 @@ def _pack_q4_0(nibbles: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
def _dequant_bank(packed: torch.Tensor, K: int, dev) -> torch.Tensor:
"""[S, OUT, K//32*18] packed Q4_0 -> [S, OUT, K] bf16 (storage order == elem order)."""
- from freetoken.models.gguf.dequant import GGML_Q4_0, dequantize
+ from freetoken.third_party.gguf import GGMLQuantizationType, quants
S, OUT, _ = packed.shape
- flat = dequantize(packed.reshape(-1), GGML_Q4_0, torch.bfloat16)
- return flat.reshape(S, OUT, K).to(dev)
+ values = quants.dequantize(packed.numpy(), GGMLQuantizationType.Q4_0)
+ return torch.from_numpy(values).reshape(S, OUT, K).to(dev, torch.bfloat16)
def _make_q4_0_cache(L, E, H, I, seed=0):
@@ -115,8 +115,10 @@ def test_cpu_decode_q4_0_matches_dequant_then_gpu(bs):
def test_cpu_decode_q4_0_matches_ggml_mmvq():
"""Sanity: the CPU W4A16 GEMV lands close to the GPU ggml MMVQ (W4A8) kernel the
offload path uses -- a looser tol since MMVQ quantizes activations to int8."""
+ from freetoken.kernel import ggml
+ from freetoken.layers.activation import gated_act_and_mul
from freetoken.moe.cpu_executor import CpuMoeExecutor
- from freetoken.moe.fused_q4_0 import fused_experts_gguf_q4_0
+ from freetoken.third_party.gguf import GGMLQuantizationType
torch.manual_seed(77)
L, E, H, I, top_k = 2, 16, 2816, 704, 8
@@ -137,11 +139,12 @@ def test_cpu_decode_q4_0_matches_ggml_mmvq():
torch.cuda.synchronize()
b = cache.bank_sources
- gate_up_layer = b["gate_up"][layer].to(dev)
- down_layer = b["down"][layer].to(dev)
- gpu_out = fused_experts_gguf_q4_0(
- hidden, gate_up_layer, down_layer, w, ids.clone(), "gelu_tanh"
- ).float()
+ q4_0 = int(GGMLQuantizationType.Q4_0)
+ h = ggml.mul_mat_id(b["gate_up"][layer].to(dev), q4_0, hidden.float(), ids)
+ inter = torch.empty(bs, top_k, I, device=dev)
+ gated_act_and_mul("gelu_tanh", h.view(-1, 2 * I), inter.view(-1, I))
+ y = ggml.mul_mat_id(b["down"][layer].to(dev), q4_0, inter, ids)
+ gpu_out = ggml.moe_weighted_sum(y, w)
rel = (cpu_out - gpu_out).abs().max() / (gpu_out.abs().max() + 1e-6)
assert rel < 6e-2, f"q4_0 vs ggml-mmvq rel err {rel.item()}"
diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.py
index d3c8d9d8..a5a7adb9 100644
--- a/tests/moe/test_offload.py
+++ b/tests/moe/test_offload.py
@@ -681,7 +681,6 @@ def test_slot_pools_of_two_bank_layouts_keep_their_rows_through_prefill():
check_resident()
decode(20)
- cache.zero_inactive_route_slots = True
cache.begin_prefill()
for layer in range(L):
cache.prefetch_prefill_layer(layer)
@@ -735,7 +734,6 @@ def test_slot_pools_split_layers_of_one_bank_layout_by_quant_tag():
cache.set_bank_sources(sources, layer_tags=tags)
assert cache.bank_caches["down"].shape == (20, 32, 144)
# the staging buffers cover pool 0 and slots [0, 6) of pool 1, so both slot 0 rows end up with another quant's bytes
- cache.zero_inactive_route_slots = True
cache.begin_prefill()
for layer in range(L):
cache.prefetch_prefill_layer(layer)
@@ -748,6 +746,56 @@ def test_slot_pools_split_layers_of_one_bank_layout_by_quant_tag():
assert not view[0].any()
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
[email protected]("wide", [(), (1, 3)], ids=["cpu-mode", "two-layouts"])
+def test_slot_pools_and_staging_buffers_end_in_zero_mmq_tails(wide):
+ from freetoken.layers.quantization.ggml_types import mmq_tail_bytes, row_bytes
+ from freetoken.moe.offload_cache import OffloadMoeCache
+ from freetoken.third_party.gguf import GGMLQuantizationType
+
+ _init_tp()
+ torch.manual_seed(0)
+ L, E, top_k = 4, 8, 4
+
+ def layout(ggml_type):
+ """(rows, row bytes, tail bytes) of gate_up and down for Gemma-4 26B-A4B's K = 2816 and 704, with fewer rows."""
+ return tuple((rows, row_bytes(ggml_type, k), mmq_tail_bytes(ggml_type, k)) for rows, k in ((32, 2816), (64, 704)))
+
+ q4_0, q8_0 = layout(GGMLQuantizationType.Q4_0), layout(GGMLQuantizationType.Q8_0)
+ shapes = [q8_0 if layer in wide else q4_0 for layer in range(L)]
+ roles = ("gate_up", "down")
+ sources = {role: [torch.randint(1, 255, (E, *shapes[l][k][:2]), dtype=torch.uint8).pin_memory() for l in range(L)] for k, role in enumerate(roles)}
+ tails = {role: [shapes[l][k][2] for l in range(L)] for k, role in enumerate(roles)}
+ # cpu mode: the slots are exactly the two staging buffers
+ pools = dict(pool_of_layer=[2, 0, 3, 1], pool_sizes=[6, 5, 12, 10], prefill_hit_d2d=True) if wide else dict(pool_of_layer=[0] * L, pool_sizes=[2 * E])
+ cache = OffloadMoeCache(num_layers=L, num_experts=E, cache_size=sum(pools["pool_sizes"]), device=torch.device("cuda"), prefill_overlap=True, **pools)
+ cache.set_bank_sources(sources, row_tail_bytes=tails)
+ assert cache.prefill_overlap
+
+ def assert_zero_tails(layer: int, views) -> None:
+ torch.cuda.synchronize()
+ for k, ((_, bank), view) in enumerate(zip(cache.banks, views)):
+ end, tail = view.storage_offset() + view.numel(), shapes[layer][k][2]
+ assert end + tail <= bank.numel() and not bank[end : end + tail].any(), (layer, k)
+
+ for _ in range(20):
+ for layer in range(L):
+ cache.ensure_experts(layer, torch.randperm(E)[:top_k].to(torch.int32).view(1, top_k).cuda())
+ cache.copy_missing()
+ cache.begin_prefill()
+ for layer in range(L):
+ cache.prefetch_prefill_layer(layer)
+ cache.prefetch_prefill_layer(layer + 1)
+ assert_zero_tails(layer, cache.wait_prefill_layer(layer))
+ cache.release_prefill_layer(layer)
+ for layer in range(L):
+ assert_zero_tails(layer, cache.pool_bank_views(layer))
+ if wide:
+ cache.materialize_layer(0)
+ cache.copy_missing()
+ assert_zero_tails(0, cache.stage_views(0, 0))
+
+
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
def test_offload_cache_rebuild_frees_the_old_slots_first():
from freetoken.moe.offload_cache import OffloadMoeCache