Skip to content

LLM inference in C/C++

C++ ggml

model: add Kimi-K3 text model (#26185)

Piotr Wilkin (ilintar)ad1de39e0708
diff --git a/common/chat.cpp b/common/chat.cpp
index a45c51fa0c..88ca2dc535 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -2325,6 +2325,179 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
return data;
}
+// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros:
+// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|>
+// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|>
+// the generation prompt already opens the think (or response) section, so the
+// section opener is optional here - same as Kimi K2 Thinking
+static common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl,
+ const autoparser::generation_params & inputs) {
+ common_chat_params data;
+
+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
+ data.supports_thinking = true;
+
+ const std::string SEP = "<|sep|>";
+ const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>";
+ const std::string THINK_START = "<|open|>think<|sep|>";
+ const std::string THINK_END = "<|close|>think<|sep|>";
+ const std::string RESP_START = "<|open|>response<|sep|>";
+ const std::string RESP_END = "<|close|>response<|sep|>";
+ const std::string TOOLS_START = "<|open|>tools<|sep|>";
+ const std::string TOOLS_END = "<|close|>tools<|sep|>";
+ const std::string CALL_START = "<|open|>call tool=\"";
+ const std::string CALL_END = "<|close|>call<|sep|>";
+ const std::string ARG_START = "<|open|>argument key=\"";
+ const std::string ARG_END = "<|close|>argument<|sep|>";
+ const std::string MSG_END = "<|close|>message<|sep|>";
+ const std::string EOM_TOKEN = "<|end_of_msg|>";
+
+ // only the markers are special tokens. tag names ("think", "response", ...) are
+ // normal tokens and must not be preserved, or prose with those words is broken
+ data.preserved_tokens = {
+ "<|open|>",
+ "<|close|>",
+ "<|sep|>",
+ "<|end_of_msg|>",
+ };
+
+ data.thinking_start_tag = THINK_START;
+ data.thinking_end_tags = { THINK_END };
+
+ // per-role message-start delimiters. user/assistant messages only have the role
+ // attribute, so the full opener is used. system and tool messages have more
+ // attributes, so those delimiters stop after the closing quote of the role
+ data.message_delimiters = {
+ { COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" },
+ { COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" },
+ { COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" },
+ { COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" },
+ };
+
+ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+ auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
+
+ if (inputs.has_continuation()) {
+ const auto & msg = inputs.continue_msg;
+
+ data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content;
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+ data.generation_prompt += THINK_END + RESP_START + msg.render_content();
+ }
+
+ data.prompt += data.generation_prompt;
+ }
+
+ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ auto end = p.end();
+
+ auto start = p.optional(p.literal(MSG_START));
+
+ // the think section is always consumed, even with reasoning extraction off:
+ // the generation prompt ends with open_tag('think'), so it is always present.
+ // reasoning stops at its own closer, or at the response opener if the model
+ // skips the closer
+ auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) :
+ p.content(p.until_one_of({ THINK_END, RESP_START }));
+
+ auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body +
+ p.optional(p.literal(THINK_END)));
+
+ // content runs to the response closer, or to the next section if truncated
+ auto response = p.optional(p.literal(RESP_START)) +
+ p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) +
+ p.optional(p.literal(RESP_END));
+
+ // the EOG token after the message closer reaches the parser as text,
+ // so it must be consumed or the parse stays incomplete
+ auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN));
+
+ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
+ return start + reasoning + response + trailer + end;
+ }
+
+ auto tool_choices = p.choice();
+ foreach_function(inputs.tools, [&](const json & tool) {
+ const auto & function = tool.at("function");
+ std::string name = function.at("name");
+ const json schema = function.contains("parameters") ? function.at("parameters") : json::object();
+
+ // arguments come one tag per key, with the JSON type in a type="..."
+ // attribute. the type is taken from the tool schema instead, as it tells
+ // us if the value is JSON or a literal string
+ auto args = p.eps();
+ if (schema.contains("properties") && !schema.at("properties").empty()) {
+ auto arg_choices = p.choice();
+ for (const auto & prop : schema.at("properties").items()) {
+ const std::string & key = prop.key();
+
+ std::string type = "string";
+ if (prop.value().is_object() && prop.value().contains("type") &&
+ prop.value().at("type").is_string()) {
+ type = prop.value().at("type").get<std::string>();
+ }
+
+ auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) :
+ p.tool_arg_value(p.until(ARG_END));
+
+ // skip the trailing type="..." attribute: anything up to <|sep|>
+ arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key,
+ p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) +
+ p.tool_arg_name(p.literal(key)) + p.literal("\"") +
+ p.until(SEP) + p.literal(SEP) + value +
+ p.tool_arg_close(p.literal(ARG_END))));
+ }
+ args = p.zero_or_more(arg_choices);
+ }
+
+ // skip the trailing index="N" attribute the same way
+ auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") +
+ p.until(SEP) + p.literal(SEP)) +
+ p.tool_args(args) + p.tool_close(p.literal(CALL_END)));
+
+ tool_choices |= p.rule("kimi-k3-tool-" + name, call);
+ });
+
+ // all calls go inside one tools section, then the message is closed. the
+ // message closer is part of the trigger rule, or else the lazy grammar
+ // rejects it once tool calls have started
+ auto tools_section =
+ p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) +
+ p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) +
+ p.optional(p.literal(EOM_TOKEN)));
+
+ auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section :
+ p.optional(tools_section);
+
+ return start + reasoning + response + tools + trailer + end;
+ });
+
+ data.parser = parser.save();
+
+ if (include_grammar) {
+ data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {
+ foreach_function(inputs.tools, [&](const json & tool) {
+ const auto & function = tool.at("function");
+ if (function.contains("parameters")) {
+ auto schema = function.at("parameters");
+ builder.resolve_refs(schema);
+ }
+ });
+ parser.build_grammar(builder, data.grammar_lazy);
+ });
+
+ data.grammar_triggers = {
+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START },
+ };
+ }
+
+ return data;
+}
+
// Cohere2 MoE (a.k.a. "North Code") parser.
//
// The assistant turn is fully marker-wrapped:
@@ -3293,6 +3466,13 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
return common_chat_params_init_kimi_k2(tmpl, params);
}
+ // Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it
+ if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos &&
+ src.find("<|end_of_msg|>") != std::string::npos) {
+ LOG_DBG("Using specialized template: Kimi K3\n");
+ return common_chat_params_init_kimi_k3(tmpl, params);
+ }
+
// Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and
// <|START_ACTION|> JSON tool calls. <|START_TEXT|> is unique to this template (the older
// Command-R templates use <|START_RESPONSE|>).
diff --git a/conversion/__init__.py b/conversion/__init__.py
index f4d475de7f..a4db77e26c 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -125,6 +125,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"JinaEmbeddingsV5Model": "bert",
"KORMoForCausalLM": "qwen",
"KimiK25ForConditionalGeneration": "deepseek",
+ "KimiK3ForConditionalGeneration": "kimi_k3",
"KimiLinearForCausalLM": "kimi_linear",
"KimiLinearModel": "kimi_linear",
"KimiVLForConditionalGeneration": "deepseek",
diff --git a/conversion/base.py b/conversion/base.py
index 718d539449..4f4fbca347 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -658,6 +658,43 @@ class ModelBase:
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
return ()
+ @staticmethod
+ def repack_mxfp4_blocks(packed: Tensor, scale: Tensor) -> np.ndarray:
+ """
+ Repack 4-bit MX weights into ggml `block_mxfp4`. Lossless - only moves bits.
+
+ Source (compressed-tensors "mxfp4-pack-quantized", also used by DeepSeek-V4):
+ packed uint8 [rows, cols/2] element 2i in the low nibble, 2i+1 in the high one
+ scale uint8 [rows, cols/32] one E8M0 biased exponent per 32-element group
+
+ Destination, per group: one scale byte then 16 code bytes, where byte j holds
+ element j in the low nibble and element j+16 in the high one.
+
+ The 4-bit codes need no remapping: both sides index into ggml's kvalues_mxfp4
+ order. ggml doubles the kvalues and halves the scale, so the value is the same.
+ """
+ p = packed.contiguous().view(torch.uint8)
+ s = scale.contiguous().view(torch.uint8)
+
+ rows, packed_cols = p.shape
+ cols = packed_cols * 2
+ if cols % 32 != 0:
+ raise ValueError(f"MXFP4 source row has {cols} values, expected a multiple of 32")
+
+ n_blocks = cols // 32
+ if tuple(s.shape) != (rows, n_blocks):
+ raise ValueError(f"MXFP4 scale shape {tuple(s.shape)} does not match {(rows, n_blocks)}")
+
+ src = p.reshape(rows, n_blocks, 16)
+ lo = src & 0x0F # elements 0, 2, 4, ...
+ hi = (src >> 4) & 0x0F # elements 1, 3, 5, ...
+
+ vals = torch.stack((lo, hi), dim=-1).reshape(rows, n_blocks, 32)
+ qs = vals[:, :, :16] | (vals[:, :, 16:] << 4)
+
+ raw = torch.cat((s.unsqueeze(-1), qs.to(torch.uint8)), dim=-1)
+ return raw.reshape(rows, n_blocks * 17).cpu().numpy()
+
@staticmethod
def _nvfp4_pack(weight: Tensor, scale: Tensor) -> tuple[np.ndarray, list[int]]:
"""Repack NVFP4 ModelOpt tensors into ggml super-block layout.
@@ -2661,7 +2698,10 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
# Step3-VL keeps text config under text_config but uses a custom top-level architecture.
# For text conversion we route to a dedicated text-only class.
# TODO: refactor this later to avoid adding exception here
- if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration"):
+ # Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
+ # Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
+ # latent MoE, situ, ...). Route on the top-level architecture instead.
+ if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"):
return arch
# if "architectures" is found in the sub-config, use that instead
diff --git a/conversion/deepseek.py b/conversion/deepseek.py
index 1846ca4010..dbfa96c2ee 100644
--- a/conversion/deepseek.py
+++ b/conversion/deepseek.py
@@ -709,31 +709,6 @@ class DeepseekV4Model(TextModel):
for name in tensors_to_remove:
del self.model_tensors[name]
- @staticmethod
- def _pack_mxfp4_blocks(weight: Tensor, scale: Tensor) -> np.ndarray:
- packed = weight.contiguous().view(torch.uint8)
- scale_u8 = scale.contiguous().view(torch.uint8)
-
- out_features, packed_cols = packed.shape
- logical_cols = packed_cols * 2
- if logical_cols % 32 != 0:
- raise ValueError(f"MXFP4 source row has {logical_cols} values, expected a multiple of 32")
-
- n_blocks = logical_cols // 32
- if tuple(scale_u8.shape) != (out_features, n_blocks):
- raise ValueError(f"MXFP4 scale shape {tuple(scale_u8.shape)} does not match {(out_features, n_blocks)}")
-
- src = packed.reshape(out_features, n_blocks, 16)
- low = src & 0x0F
- high = (src >> 4) & 0x0F
-
- # The safetensors bytes store adjacent values as low/high nibbles.
- # ggml MXFP4 blocks store values 0..15 in low nibbles and 16..31 in high nibbles.
- vals = torch.stack((low, high), dim=-1).reshape(out_features, n_blocks, 32)
- qs = vals[:, :, :16] | (vals[:, :, 16:] << 4)
- raw = torch.cat((scale_u8.unsqueeze(-1), qs.to(torch.uint8)), dim=-1)
- return raw.reshape(out_features, n_blocks * 17).cpu().numpy()
-
def _write_mxfp4_expert_tensor(self, bid: int, proj: str, tensor_key: gguf.MODEL_TENSOR) -> list[str]:
n_experts = self.hparams["n_routed_experts"]
data: np.ndarray | None = None
@@ -747,7 +722,7 @@ class DeepseekV4Model(TextModel):
weight = LazyTorchTensor.to_eager(self.model_tensors[weight_name]())
scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]())
- packed = self._pack_mxfp4_blocks(weight, scale)
+ packed = self.repack_mxfp4_blocks(weight, scale)
if data is None:
data = np.empty((n_experts, *packed.shape), dtype=packed.dtype)
data[eid] = packed
diff --git a/conversion/kimi_k3.py b/conversion/kimi_k3.py
new file mode 100644
index 0000000000..a5005e15bd
--- /dev/null
+++ b/conversion/kimi_k3.py
@@ -0,0 +1,375 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Callable, Iterable, Iterator, TYPE_CHECKING
+
+import numpy as np
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import LazyTorchTensor, ModelBase, TextModel, gguf, logger
+
+from .kimi_linear import KimiLinearModel
+
+
[email protected]("KimiK3ForConditionalGeneration")
+class KimiK3Model(TextModel):
+ """
+ Kimi-K3 text model (KimiLinearForCausalLM under a `language_model.` prefix).
+
+ Shares the hybrid MLA + KDA skeleton with kimi-linear, but that converter
+ cannot load it: K3 adds cross-layer attention residuals, a latent MoE, the
+ situ activation, an MLA output gate and a full-rank KDA gate.
+
+ The vision tower and mm_projector are skipped - text only for now.
+ """
+
+ model_arch = gguf.MODEL_ARCH.KIMI_K3
+
+ _experts: list[dict[str, Tensor]] | None = None
+
+ # `<x>_res_norm.weight` and `<x>_res_proj.weight` are only used as their
+ # elementwise product, so they are fused into one [n_embd] vector here.
+ # they arrive apart, so buffer the first one and tag it with its kind.
+ _res_parts: dict[str, tuple[str, Tensor]]
+
+ # HF suffix -> (gguf tensor, per-layer?)
+ _RES_FUSIONS = {
+ "self_attention_res": (gguf.MODEL_TENSOR.ATTN_RES_SCORE, True),
+ "mlp_res": (gguf.MODEL_TENSOR.FFN_RES_SCORE, True),
+ "output_attn_res": (gguf.MODEL_TENSOR.OUTPUT_RES_SCORE, False),
+ }
+
+ # compressed-tensors MXFP4. the `language_model.` prefix is still there, as
+ # self.model_tensors is keyed by the raw checkpoint names
+ _MXFP4_FORMAT = "mxfp4-pack-quantized"
+ _MXFP4_EXPERT_RE = re.compile(
+ r"^(?:language_model\.)?model\.layers\.(\d+)"
+ r"\.block_sparse_moe\.experts\.(\d+)\.(w[123])\.weight_packed$"
+ )
+ _MXFP4_PROJ = {
+ "w1": gguf.MODEL_TENSOR.FFN_GATE_EXP,
+ "w2": gguf.MODEL_TENSOR.FFN_DOWN_EXP,
+ "w3": gguf.MODEL_TENSOR.FFN_UP_EXP,
+ }
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._res_parts = {}
+
+ def set_vocab(self):
+ # K3 has the same TikToken vocab as K2, so kimi-linear's vocab handling works.
+ # borrowed, not inherited: the method only touches TextModel members, and K3
+ # shares none of kimi-linear's tensor layout.
+ KimiLinearModel.set_vocab(self) # ty: ignore[invalid-argument-type]
+
+ # ...but that forces eos to the tokenizer's eos_id, which is [EOS], the
+ # document terminator. K3's config says <|end_of_msg|>, the turn terminator;
+ # with [EOS] the generation never stops at the end of a turn.
+ if (eos := self.hparams.get("eos_token_id")) is not None:
+ logger.info(f"restoring configured eos_token_id {eos} (kimi-linear forces the tokenizer's)")
+ self.gguf_writer.add_eos_token_id(eos)
+
+ # K3 renders chats in python (encoding_k3.py) and ships no jinja template,
+ # so add the bundled one when the model has none
+ if gguf.SpecialVocab(self.dir_model, load_merges=False).chat_template is None:
+ template_path = Path(__file__).parent.parent / "models" / "templates" / "Kimi-K3.jinja"
+ logger.info(f"gguf: model has no chat template, using {template_path.name}")
+ self.gguf_writer.add_chat_template(template_path.read_text(encoding="utf-8"))
+
+ #
+ # compressed-tensors MXFP4 -> ggml MXFP4
+ #
+
+ def _is_mxfp4_packed(self) -> bool:
+ quant_config = self.hparams.get("quantization_config") or {}
+ return (quant_config.get("quant_method") == "compressed-tensors"
+ and quant_config.get("format") == self._MXFP4_FORMAT)
+
+ def dequant_model(self):
+ if not self._is_mxfp4_packed():
+ return super().dequant_model()
+
+ # skipping base.py's dequant is only safe if the experts are the only
+ # quantized tensors, so check it
+ stray = [n for n in self.model_tensors
+ if n.endswith(".weight_packed") and not self._MXFP4_EXPERT_RE.match(n)]
+ if stray:
+ raise NotImplementedError(
+ f"{len(stray)} MXFP4 tensor(s) outside the routed experts, e.g. {stray[0]!r}; "
+ "only the routed experts have a repack path"
+ )
+
+ def _mxfp4_expert_tensor(self, loaders: list[tuple[Callable[[], Tensor], Callable[[], Tensor]]]):
+ """
+ One stacked [n_expert, rows, cols] MXFP4 tensor, built lazily.
+
+ gguf_writer holds every added tensor until the final write, so building
+ this eagerly (like the DeepSeek-V4 path does) keeps all ~1.38 TB of
+ experts in memory. lazy means only the tensor being written is resident.
+ """
+ # meta shapes, so this does not read any weights
+ rows, packed_cols = loaders[0][0]().shape
+ n_blocks = (packed_cols * 2) // 32
+ byte_shape = (len(loaders), rows, n_blocks * 17)
+
+ def load(fns: list[tuple[Callable[[], Tensor], Callable[[], Tensor]]]) -> np.ndarray:
+ out = np.empty(byte_shape, dtype=np.uint8)
+ for eid, (packed_fn, scale_fn) in enumerate(fns):
+ out[eid] = self.repack_mxfp4_blocks(
+ LazyTorchTensor.to_eager(packed_fn()),
+ LazyTorchTensor.to_eager(scale_fn()),
+ )
+ return out
+
+ # loaders goes through args, not the closure, so that `func` matches
+ # LazyBase's single-argument shape
+ return gguf.LazyNumpyTensor(
+ meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(np.uint8, byte_shape),
+ args=(loaders,),
+ func=load,
+ )
+
+ def _write_mxfp4_experts(self) -> None:
+ n_experts = self.hparams["num_experts"]
+
+ # (bid, wid) -> {expert id: (packed name, scale name)}
+ groups: dict[tuple[int, str], dict[int, tuple[str, str]]] = {}
+ for name in self.model_tensors:
+ m = self._MXFP4_EXPERT_RE.match(name)
+ if m is None:
+ continue
+ bid, eid, wid = int(m.group(1)), int(m.group(2)), m.group(3)
+ scale_name = name.removesuffix("_packed") + "_scale"
+ if scale_name not in self.model_tensors:
+ raise KeyError(f"missing {scale_name} for {name}")
+ groups.setdefault((bid, wid), {})[eid] = (name, scale_name)
+
+ consumed: list[str] = []
+ for (bid, wid), experts in sorted(groups.items()):
+ missing = [e for e in range(n_experts) if e not in experts]
+ if missing:
+ raise KeyError(
+ f"layer {bid} {wid}: {len(missing)} of {n_experts} experts missing, "
+ f"first is {missing[0]}"
+ )
+ if len(experts) != n_experts:
+ raise KeyError(f"layer {bid} {wid}: {len(experts)} experts, expected {n_experts}")
+
+ loaders = []
+ for eid in range(n_experts):
+ packed_name, scale_name = experts[eid]
+ loaders.append((self.model_tensors[packed_name], self.model_tensors[scale_name]))
+ consumed += [packed_name, scale_name]
+
+ data = self._mxfp4_expert_tensor(loaders)
+ new_name = self.format_tensor_name(self._MXFP4_PROJ[wid], bid)
+ shape = gguf.quant_shape_from_byte_shape(data.shape, gguf.GGMLQuantizationType.MXFP4)
+ logger.info(
+ f"{new_name}: repacked {n_experts} experts to MXFP4, "
+ f"shape = {{{', '.join(str(n) for n in reversed(shape))}}}"
+ )
+ self.gguf_writer.add_tensor(new_name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP4)
+
+ for name in consumed:
+ del self.model_tensors[name]
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # not a generator on purpose: base.py chains this with get_tensors(), so the
+ # tensors used here must be removed from model_tensors before that starts
+ if self._is_mxfp4_packed():
+ self._write_mxfp4_experts()
+ return ()
+
+ def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
+ for name, data in super().get_tensors():
+ if name.startswith(("vision_tower.", "mm_projector.")):
+ continue # text only
+ if name.startswith("language_model."):
+ name = name[len("language_model."):]
+ yield name, data
+
+ def set_gguf_parameters(self):
+ # MLA is served as MQA with a single large head, then decompressed
+ self.hparams["num_key_value_heads"] = 1
+
+ super().set_gguf_parameters()
+ self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])
+
+ linear_attn_config = self.hparams["linear_attn_config"]
+
+ # n_head_kv == 0 marks a KDA (recurrent) layer. the layer lists are 1-indexed,
+ # as KimiLinearConfig.is_kda_layer uses (layer_idx + 1)
+ full_attn_layers = linear_attn_config["full_attn_layers"]
+ n_kv_heads = [
+ self.hparams["num_key_value_heads"] if (il + 1) in full_attn_layers else 0
+ for il in range(self.hparams["num_hidden_layers"])
+ ]
+ assert len(n_kv_heads) == self.hparams["num_hidden_layers"]
+ self.gguf_writer.add_head_count_kv(n_kv_heads)
+
+ # --- KDA ---
+ self.gguf_writer.add_ssm_conv_kernel(linear_attn_config["short_conv_kernel_size"])
+ self.gguf_writer.add_kda_head_dim(linear_attn_config["head_dim"])
+ if (lb := linear_attn_config.get("gate_lower_bound")) is not None:
+ self.gguf_writer.add_kda_gate_lower_bound(lb)
+
+ # --- MLA ---
+ if (q_lora_rank := self.hparams.get("q_lora_rank")) is not None:
+ self.gguf_writer.add_q_lora_rank(q_lora_rank)
+ kv_lora_rank = self.hparams["kv_lora_rank"]
+ self.gguf_writer.add_kv_lora_rank(kv_lora_rank)
+
+ qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
+ qk_rope_head_dim = self.hparams["qk_rope_head_dim"]
+ v_head_dim = self.hparams["v_head_dim"]
+ # K3 is nope-only; qk_rope_head_dim still sizes the un-absorbed part of K
+ assert self.hparams.get("mla_use_nope"), "K3 MLA is expected to be nope-only"
+ self.gguf_writer.add_rope_dimension_count(qk_rope_head_dim)
+ # MLA is served as MQA, so the cache holds the compressed latent
+ self.gguf_writer.add_key_length(kv_lora_rank + qk_rope_head_dim)
+ self.gguf_writer.add_value_length(kv_lora_rank)
+ self.gguf_writer.add_key_length_mla(qk_nope_head_dim + qk_rope_head_dim)
+ self.gguf_writer.add_value_length_mla(v_head_dim)
+
+ # --- MoE ---
+ self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
+ self.gguf_writer.add_expert_shared_count(self.hparams["num_shared_experts"])
+ self.gguf_writer.add_leading_dense_block_count(self.hparams["first_k_dense_replace"])
+ self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
+ self.gguf_writer.add_expert_weights_norm(self.hparams["moe_renormalize"])
+ assert self.hparams["moe_router_activation_func"] == "sigmoid"
+ self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+ # latent MoE: routed experts live in a down-projected space
+ if (latent := self.hparams.get("routed_expert_hidden_size")) is not None:
+ self.gguf_writer.add_expert_latent_length(latent)
+
+ # --- situ activation ---
+ assert self.hparams["hidden_act"] == "situ", \
+ f"unexpected hidden_act {self.hparams['hidden_act']!r}"
+ self.gguf_writer.add_activation_situ_beta(self.hparams["activation_situ_beta"])
+ self.gguf_writer.add_activation_situ_linear_beta(self.hparams["activation_situ_linear_beta"])
+
+ # --- cross-layer attention residuals ---
+ self.gguf_writer.add_attn_res_block_size(self.hparams["attn_res_block_size"])
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+ if self._experts is not None:
+ leftover = [k for d in self._experts for k in d.keys()]
+ if leftover:
+ raise ValueError(f"Unprocessed experts: {leftover}")
+ if self._res_parts:
+ raise ValueError(f"Unpaired attention-residual tensors: {sorted(self._res_parts)}")
+ if self._is_mxfp4_packed():
+ # label the file for what it is; prepare_metadata runs after this
+ self._is_mxfp4 = True
+ self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE
+
+ def _try_fuse_res(self, data_torch: Tensor, name: str, bid: int | None):
+ """
+ Pair <x>_res_norm.weight with <x>_res_proj.weight and emit their product.
+
+ Returns None if this is not a res tensor, [] if buffered until its pair.
+ """
+ for prefix, (tensor_id, per_layer) in self._RES_FUSIONS.items():
+ for kind in ("norm", "proj"):
+ if not name.endswith(f"{prefix}_{kind}.weight"):
+ continue
+ key = f"{prefix}.{bid}"
+ other = self._res_parts.pop(key, None)
+ if other is None:
+ self._res_parts[key] = (kind, data_torch)
+ return []
+ other_kind, other_data = other
+ assert other_kind != kind, f"duplicate {kind} for {key}"
+ norm = data_torch if kind == "norm" else other_data
+ proj = data_torch if kind == "proj" else other_data
+ fused = norm.float().flatten() * proj.float().flatten()
+ # ".weight" suffix matches the convention map_tensor_name applies
+ new_name = (self.format_tensor_name(tensor_id, bid) if per_layer
+ else gguf.TENSOR_NAMES[tensor_id] + ".weight")
+ logger.info(f"fused {prefix}_norm * {prefix}_proj -> {new_name}")
+ return [(new_name, fused)]
+ return None
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # --- cross-layer attention residuals: fuse norm * proj ---
+ fused = self._try_fuse_res(data_torch, name, bid)
+ if fused is not None:
+ yield from fused
+ return
+
+ # --- KDA conv1d: HF [d_inner, 1, d_conv] -> ggml ne [d_conv, 1, d_inner, 1] ---
+ # GGUF reverses the numpy shape on write, so target numpy (1, d_inner, 1, d_conv).
+ # conv_step varies fastest in both layouts, so this is a pure reshape.
+ if name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")):
+ if data_torch.ndim == 3: # [d_inner, 1, d_conv]
+ d_inner, _, d_conv = data_torch.shape
+ elif data_torch.ndim == 2: # [d_inner, d_conv]
+ d_inner, d_conv = data_torch.shape
+ else:
+ raise ValueError(f"unexpected conv1d rank {data_torch.ndim} for {name}")
+ data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
+
+ # -exp(A_log) is folded here so the graph does not have to
+ if name.endswith(".A_log"):
+ n_head = self.hparams["num_attention_heads"]
+ data_torch = -torch.exp(data_torch.float()[:n_head])
+
+ # dt_bias -> the name SSM_DT's mapping expects
+ if name.endswith(".dt_bias"):
+ name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
+
+ # --- g_proj is two different tensors sharing one HF name ---
+ # KDA layers: full-rank gate, [d_inner, n_embd] (replaces g_a/g_b)
+ # MLA layers: output gate, [n_head*v_head_dim, n_embd]
+ # Name-based mapping cannot tell them apart, so resolve by layer type.
+ if name.endswith(".self_attn.g_proj.weight"):
+ assert bid is not None
+ is_kda = (bid + 1) not in self.hparams["linear_attn_config"]["full_attn_layers"]
+ tensor_id = gguf.MODEL_TENSOR.SSM_G if is_kda else gguf.MODEL_TENSOR.ATTN_GATE
+ yield self.format_tensor_name(tensor_id, bid), data_torch
+ return
+
+ # --- routed experts: stack per-expert 2D weights into one 3D tensor ---
+ if ".block_sparse_moe.experts." in name:
+ n_experts = self.hparams["num_experts"]
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) < n_experts * 3:
+ return
+
+ # w1: gate, w2: down, w3: up
+ for wid, tensor_id in (("w1", gguf.MODEL_TENSOR.FFN_GATE_EXP),
+ ("w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP),
+ ("w3", gguf.MODEL_TENSOR.FFN_UP_EXP)):
+ datas = []
+ for xid in range(n_experts):
+ ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight"
+ datas.append(self._experts[bid].pop(ename))
+ stacked = torch.stack(datas, dim=0)
+ yield from super().modify_tensors(stacked, self.format_tensor_name(tensor_id, bid), bid)
+ return
+
+ # --- MLA absorption: split kv_b into k_b (transposed) and v_b ---
+ if name.endswith("kv_b_proj.weight"):
+ n_head_kv = self.hparams["num_key_value_heads"]
+ v_head_dim = self.hparams["v_head_dim"]
+ qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
+ assert data_torch.shape[0] == n_head_kv * (v_head_dim + qk_nope_head_dim)
+ kv_b = data_torch.view(n_head_kv, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])
+ k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
+ k_b = k_b.transpose(1, 2)
+ yield from super().modify_tensors(k_b, name.replace("kv_b_proj", "k_b_proj"), bid)
+ yield from super().modify_tensors(v_b, name.replace("kv_b_proj", "v_b_proj"), bid)
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index a312c8aa00..e359216d7c 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -124,6 +124,7 @@ class Keys:
EXPERT_WEIGHTS_NORM = "{arch}.expert_weights_norm"
EXPERT_GATING_FUNC = "{arch}.expert_gating_func"
EXPERT_GROUP_SCALE = "{arch}.expert_group_scale"
+ EXPERT_LATENT_LENGTH = "{arch}.expert_latent_length"
EXPERTS_PER_GROUP = "{arch}.experts_per_group"
MOE_EVERY_N_LAYERS = "{arch}.moe_every_n_layers"
MOE_LATENT_SIZE = "{arch}.moe_latent_size"
@@ -238,6 +239,13 @@ class Keys:
SCALING_YARN_BETA_FAST = "{arch}.rope.scaling.yarn_beta_fast"
SCALING_YARN_BETA_SLOW = "{arch}.rope.scaling.yarn_beta_slow"
+ class Activation:
+ SITU_BETA = "{arch}.activation.situ_beta"
+ SITU_LINEAR_BETA = "{arch}.activation.situ_linear_beta"
+
+ class AttnRes:
+ BLOCK_SIZE = "{arch}.attn_res.block_size"
+
class Split:
LLM_KV_SPLIT_NO = "split.no"
LLM_KV_SPLIT_COUNT = "split.count"
@@ -252,7 +260,8 @@ class Keys:
DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms"
class KDA:
- HEAD_DIM = "{arch}.kda.head_dim"
+ HEAD_DIM = "{arch}.kda.head_dim"
+ GATE_LOWER_BOUND = "{arch}.kda.gate_lower_bound"
class WKV:
HEAD_SIZE = "{arch}.wkv.head_size"
@@ -580,6 +589,7 @@ class MODEL_ARCH(IntEnum):
LLAMA_EMBED = auto()
MAINCODER = auto()
KIMI_LINEAR = auto()
+ KIMI_K3 = auto()
TALKIE = auto()
MELLUM = auto()
NANBEIGE = auto()
@@ -698,6 +708,13 @@ class MODEL_TENSOR(IntEnum):
SSM_BETA = auto() # Kimi Linear qwen3.5
SSM_G_A = auto() # Kimi Linear
SSM_G_B = auto() # Kimi Linear
+ SSM_G = auto() # Kimi K3 (full-rank KDA gate, replaces SSM_G_A/SSM_G_B)
+ ATTN_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, pre-attention)
+ FFN_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, pre-FFN)
+ OUTPUT_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, final)
+ FFN_ROUTED_DOWN = auto() # Kimi K3 (latent MoE: hidden -> latent)
+ FFN_ROUTED_UP = auto() # Kimi K3 (latent MoE: latent -> hidden)
+ FFN_ROUTED_NORM = auto() # Kimi K3 (latent MoE: norm on expert output)
TIME_MIX_W0 = auto()
TIME_MIX_W1 = auto()
TIME_MIX_W2 = auto()
@@ -1288,6 +1305,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.LLAMA_EMBED: "llama-embed",
MODEL_ARCH.MAINCODER: "maincoder",
MODEL_ARCH.KIMI_LINEAR: "kimi-linear",
+ MODEL_ARCH.KIMI_K3: "kimi-k3",
MODEL_ARCH.TALKIE: "talkie",
MODEL_ARCH.MELLUM: "mellum",
MODEL_ARCH.NANBEIGE: "nanbeige",
@@ -1404,6 +1422,13 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.SSM_BETA: "blk.{bid}.ssm_beta", # Kimi Linear qwen3.5
MODEL_TENSOR.SSM_G_A: "blk.{bid}.ssm_g_a", # Kimi Linear
MODEL_TENSOR.SSM_G_B: "blk.{bid}.ssm_g_b", # Kimi Linear
+ MODEL_TENSOR.SSM_G: "blk.{bid}.ssm_g", # Kimi K3
+ MODEL_TENSOR.ATTN_RES_SCORE: "blk.{bid}.attn_res_score", # Kimi K3
+ MODEL_TENSOR.FFN_RES_SCORE: "blk.{bid}.ffn_res_score", # Kimi K3
+ MODEL_TENSOR.OUTPUT_RES_SCORE: "output_res_score", # Kimi K3
+ MODEL_TENSOR.FFN_ROUTED_DOWN: "blk.{bid}.ffn_routed_down", # Kimi K3
+ MODEL_TENSOR.FFN_ROUTED_UP: "blk.{bid}.ffn_routed_up", # Kimi K3
+ MODEL_TENSOR.FFN_ROUTED_NORM: "blk.{bid}.ffn_routed_norm", # Kimi K3
MODEL_TENSOR.TIME_MIX_W0: "blk.{bid}.time_mix_w0",
MODEL_TENSOR.TIME_MIX_W1: "blk.{bid}.time_mix_w1",
MODEL_TENSOR.TIME_MIX_W2: "blk.{bid}.time_mix_w2",
@@ -4960,6 +4985,56 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
],
+ MODEL_ARCH.KIMI_K3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_RES_SCORE,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_RES_SCORE,
+ MODEL_TENSOR.FFN_RES_SCORE,
+ # MLA (full-attention layers)
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_KV_A_MQA,
+ MODEL_TENSOR.ATTN_KV_B,
+ MODEL_TENSOR.ATTN_K_B,
+ MODEL_TENSOR.ATTN_V_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV_A_NORM,
+ # KDA (linear-attention layers)
+ MODEL_TENSOR.SSM_CONV1D_Q,
+ MODEL_TENSOR.SSM_CONV1D_K,
+ MODEL_TENSOR.SSM_CONV1D_V,
+ MODEL_TENSOR.SSM_F_A,
+ MODEL_TENSOR.SSM_F_B,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_G,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ # FFN
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_ROUTED_DOWN,
+ MODEL_TENSOR.FFN_ROUTED_UP,
+ MODEL_TENSOR.FFN_ROUTED_NORM,
+ ],
MODEL_ARCH.TALKIE: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT,
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 05f86396dc..0aee5168c0 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1103,6 +1103,21 @@ class GGUFWriter:
def add_ssm_dt_b_c_rms(self, value: bool) -> None:
self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value)
+ def add_kda_gate_lower_bound(self, value: float) -> None:
+ self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value)
+
+ def add_expert_latent_length(self, value: int) -> None:
+ self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value)
+
+ def add_activation_situ_beta(self, value: float) -> None:
+ self.add_float32(Keys.Activation.SITU_BETA.format(arch=self.arch), value)
+
+ def add_activation_situ_linear_beta(self, value: float) -> None:
+ self.add_float32(Keys.Activation.SITU_LINEAR_BETA.format(arch=self.arch), value)
+
+ def add_attn_res_block_size(self, value: int) -> None:
+ self.add_uint32(Keys.AttnRes.BLOCK_SIZE.format(arch=self.arch), value)
+
def add_kda_head_dim(self, value: int) -> None:
self.add_uint32(Keys.KDA.HEAD_DIM.format(arch=self.arch), value)
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index ee3a8d3aa4..393976fb61 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -912,6 +912,19 @@ class TensorNameMap:
"model.layers.{bid}.linear_attn.in_proj_b", # qwen3.5
"model.layers.{bid}.self_attn.b_proj", # Kimi Linear
),
+ # Kimi K3 latent MoE: routed experts operate in a down-projected space
+ MODEL_TENSOR.FFN_ROUTED_DOWN: (
+ "model.layers.{bid}.block_sparse_moe.routed_expert_down_proj",
+ ),
+
+ MODEL_TENSOR.FFN_ROUTED_UP: (
+ "model.layers.{bid}.block_sparse_moe.routed_expert_up_proj",
+ ),
+
+ MODEL_TENSOR.FFN_ROUTED_NORM: (
+ "model.layers.{bid}.block_sparse_moe.routed_expert_norm",
+ ),
+
MODEL_TENSOR.SSM_G_A: (
"model.layers.{bid}.self_attn.g_a_proj",
),
diff --git a/models/templates/Kimi-K3.jinja b/models/templates/Kimi-K3.jinja
new file mode 100644
index 0000000000..48de47fc90
--- /dev/null
+++ b/models/templates/Kimi-K3.jinja
@@ -0,0 +1,324 @@
+{%- macro escape_attr(value) -%}
+{{- value|string|replace('&', '&amp;')|replace('"', '&quot;') -}}
+{%- endmacro -%}
+
+{%- macro open_tag(tag, attrs=[]) -%}
+{{- '<|open|>' + tag -}}
+{%- for attr in attrs -%}
+{{- ' ' + attr[0] + '="' -}}{{- escape_attr(attr[1]) -}}{{- '"' -}}
+{%- endfor -%}
+{{- '<|sep|>' -}}
+{%- endmacro -%}
+
+{%- macro close_tag(tag) -%}
+{{- '<|close|>' + tag + '<|sep|>' -}}
+{%- endmacro -%}
+
+{%- macro next_image(state) -%}
+{%- if image_prompts is defined and image_prompts is not none -%}
+ {%- if state.image_index >= image_prompts|length -%}
+ {{- raise_exception('More image placeholders than image prompts.') -}}
+ {%- endif -%}
+ {{- image_prompts[state.image_index] -}}
+ {%- set state.image_index = state.image_index + 1 -%}
+{%- else -%}
+ {{- '<|kimi_image_placeholder|>' -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_text(text, state) -%}
+{%- set text = text|string -%}
+{%- if image_prompts is defined and image_prompts is not none and '<|kimi_image_placeholder|>' in text -%}
+ {%- set parts = text.split('<|kimi_image_placeholder|>') -%}
+ {%- for part in parts -%}
+ {{- part -}}
+ {%- if not loop.last -%}{{- next_image(state) -}}{%- endif -%}
+ {%- endfor -%}
+{%- else -%}
+ {{- text -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_content(content, state) -%}
+{%- if content is string -%}
+ {{- render_text(content, state) -}}
+{%- elif content is not none and content is defined -%}
+ {%- for part in content -%}
+ {%- if part.type in ['image', 'image_url'] -%}
+ {{- next_image(state) -}}
+ {%- else -%}
+ {{- render_text(part.text, state) -}}
+ {%- endif -%}
+ {%- endfor -%}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro internal_system_message(message_type, body) -%}
+{{- open_tag('message', [('role', 'system'), ('type', message_type)]) -}}
+{{- body|trim -}}
+{{- close_tag('message') -}}
+{{- '<|end_of_msg|>' -}}
+{%- endmacro -%}
+
+{%- macro json_sorted(value) -%}
+{#- tojson has no sort_keys, so sort each mapping level with dictsort to match the
+ reference implementation. Array order is kept as-is. -#}
+{%- if value is mapping -%}
+{{- '{' -}}
+{%- for key, item in value|dictsort -%}
+{%- if not loop.first -%}{{- ',' -}}{%- endif -%}
+{{- key|tojson(ensure_ascii=false) -}}{{- ':' -}}{{- json_sorted(item) -}}
+{%- endfor -%}
+{{- '}' -}}
+{%- elif value is string or value is number or value is boolean or value is none -%}
+{{- value|tojson(ensure_ascii=false) -}}
+{%- else -%}
+{{- '[' -}}
+{%- for item in value -%}
+{%- if not loop.first -%}{{- ',' -}}{%- endif -%}
+{{- json_sorted(item) -}}
+{%- endfor -%}
+{{- ']' -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_declare(tool_list, dynamic=false) -%}
+{{- open_tag('message', [('role', 'system'), ('type', 'tool-declare')]) -}}
+{%- if dynamic -%}
+{{- '## New Tools Available\nThe system dynamically extends the toolset via lazy-loading.\nYou have access to all existing and extended tools.\nHere are the specs for the extended tools.\n\n```json\n' -}}
+{%- else -%}
+{{- '# Tools\nHere are the available tools, described in JSONSchema.\n\n```json\n' -}}
+{%- endif -%}
+{{- json_sorted(tool_list) -}}
+{{- '\n```' -}}
+{{- close_tag('message') -}}
+{{- '<|end_of_msg|>' -}}
+{%- endmacro -%}
+
+{%- macro xtml_type(value) -%}
+{%- if value is boolean -%}boolean
+{%- elif value is none -%}null
+{%- elif value is number -%}number
+{%- elif value is string -%}string
+{%- elif value is mapping -%}object
+{%- else -%}array
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro xtml_value(value) -%}
+{%- if value is string -%}
+{{- value -}}
+{%- else -%}
+{{- value|tojson(ensure_ascii=false) -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_assistant(message, state) -%}
+{%- if thinking -%}
+ {%- set reasoning_content = message.get('reasoning_content') or message.get('reasoning') -%}
+ {{- open_tag('think') -}}
+ {%- if reasoning_content is not none and reasoning_content|string|trim -%}
+ {{- render_text(reasoning_content, state) -}}
+ {%- endif -%}
+ {{- close_tag('think') -}}
+{%- endif -%}
+{{- open_tag('response') -}}
+{{- render_content(message.get('content'), state) -}}
+{{- close_tag('response') -}}
+{%- set tool_calls = message.get('tool_calls') -%}
+{%- if tool_calls -%}
+ {{- open_tag('tools') -}}
+ {%- for tool_call in tool_calls -%}
+ {%- if tool_call is not mapping -%}
+ {{- raise_exception('Kimi K3 tool calls must be mappings.') -}}
+ {%- endif -%}
+ {%- set fn = tool_call.function if tool_call.function is defined and tool_call.function is mapping else tool_call -%}
+ {%- if fn.get('name') is none -%}
+ {{- raise_exception('Kimi K3 tool calls require a function name.') -}}
+ {%- endif -%}
+ {{- open_tag('call', [('tool', fn.name), ('index', loop.index)]) -}}
+ {%- set arguments = fn.get('arguments', {}) -%}
+ {%- set json_block = fn.get('_xtml_json_block') -%}
+ {%- if json_block is not none -%}
+ {{- open_tag('json', [('type', 'object')]) -}}
+ {{- render_text(json_block, state) -}}
+ {{- close_tag('json') -}}
+ {%- elif arguments is mapping -%}
+ {%- for key, value in arguments.items() -%}
+ {{- open_tag('argument', [('key', key), ('type', xtml_type(value))]) -}}
+ {{- render_text(xtml_value(value), state) -}}
+ {{- close_tag('argument') -}}
+ {%- endfor -%}
+ {%- elif arguments is string and arguments|trim -%}
+ {{- open_tag('json', [('type', 'object')]) -}}
+ {{- render_text(arguments, state) -}}
+ {{- close_tag('json') -}}
+ {%- elif arguments is not none and arguments is not string -%}
+ {{- raise_exception('Kimi K3 tool call arguments must be a mapping or a JSON object string.') -}}
+ {%- endif -%}
+ {{- close_tag('call') -}}
+ {%- endfor -%}
+ {{- close_tag('tools') -}}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_message(message, state, resolved_name=none) -%}
+{%- set state.tool_index = state.tool_index + 1 -%}
+{%- if resolved_name is not none -%}
+ {%- set tool_name = resolved_name -%}
+{%- elif 'tool' in message -%}
+ {%- set tool_name = message.get('tool') -%}
+{%- else -%}
+ {%- set tool_name = message.get('name') -%}
+{%- endif -%}
+{%- if tool_name is none and state.tool_calls is not none and state.tool_index <= state.tool_calls|length -%}
+ {%- set fallback_call = state.tool_calls[state.tool_index - 1] -%}
+ {%- set fallback_fn = fallback_call.function if fallback_call.function is defined and fallback_call.function is mapping else fallback_call -%}
+ {%- set tool_name = fallback_fn.name -%}
+{%- endif -%}
+{%- if tool_name is none -%}
+ {{- raise_exception('Kimi K3 tool messages need a resolvable tool name: carry `tool`/`name`, or match a preceding assistant tool_call by order.') -}}
+{%- endif -%}
+{{- open_tag('message', [('role', 'tool'), ('tool', tool_name), ('index', state.tool_index)]) -}}
+{{- render_content(message.get('content'), state) -}}
+{{- close_tag('message') -}}
+{{- '<|end_of_msg|>' -}}
+{%- endmacro -%}
+
+{%- if thinking is undefined -%}
+ {%- set thinking = true -%}
+{%- endif -%}
+{%- if thinking_effort is undefined -%}
+ {%- set thinking_effort = 'max' -%}
+{%- endif -%}
+{%- if thinking and thinking_effort is not none and thinking_effort not in ['low', 'high', 'max'] -%}
+ {{- raise_exception('Unsupported thinking_effort=' + thinking_effort|string + '; supported values are low, high, and max.') -}}
+{%- endif -%}
+
+{%- set state = namespace(image_index=0, tool_calls=none, tool_index=0, response_schema=none) -%}
+
+{%- if tools is defined and tools -%}
+ {{- render_tool_declare(tools) -}}
+{%- endif -%}
+
+{%- if thinking and thinking_effort in ['low', 'high', 'max'] -%}
+ {{- internal_system_message(
+ 'thinking-effort',
+ '`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.\nNow the system is invoked with `thinking_effort=' + thinking_effort|string + '`.'
+ ) -}}
+{%- endif -%}
+
+{%- for message in messages -%}
+ {%- if message is mapping -%}
+ {%- if 'role' not in message -%}
+ {{- raise_exception('Kimi K3 messages require a role.') -}}
+ {%- elif message.role == 'user' -%}
+ {%- set attrs = [('role', 'user')] -%}
+ {%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
+ {{- open_tag('message', attrs) -}}
+ {{- render_content(message.get('content'), state) -}}
+ {{- close_tag('message') -}}
+ {{- '<|end_of_msg|>' -}}
+ {%- elif message.role == 'system' and message.get('tools') -%}
+ {{- render_tool_declare(message.tools, dynamic=true) -}}
+ {%- elif message.role == 'system' -%}
+ {%- set attrs = [('role', 'system')] -%}
+ {%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
+ {{- open_tag('message', attrs) -}}
+ {{- render_content(message.get('content'), state) -}}
+ {{- close_tag('message') -}}
+ {{- '<|end_of_msg|>' -}}
+ {%- elif message.role == 'assistant' -%}
+ {%- set state.tool_calls = message.get('tool_calls') -%}
+ {%- set state.tool_index = 0 -%}
+ {%- set attrs = [('role', 'assistant')] -%}
+ {%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
+ {{- open_tag('message', attrs) -}}
+ {{- render_assistant(message, state) -}}
+ {{- close_tag('message') -}}
+ {{- '<|end_of_msg|>' -}}
+ {%- elif message.role == 'tool' and (loop.first or messages[loop.index0 - 1].role != 'tool') -%}
+ {%- set run = namespace(tool_messages=[], resolved_count=0) -%}
+ {%- for candidate in messages[loop.index0:] -%}
+ {%- if candidate is not mapping or candidate.role != 'tool' -%}{%- break -%}{%- endif -%}
+ {%- set run.tool_messages = run.tool_messages + [candidate] -%}
+ {%- set call_id = candidate.get('tool_call_id', candidate.get('id')) -%}
+ {%- set match = namespace(found=false) -%}
+ {%- if call_id is not none and state.tool_calls is not none -%}
+ {%- for tool_call in state.tool_calls -%}
+ {%- if not match.found and tool_call is mapping and tool_call.get('id') is not none and tool_call.get('id')|string == call_id|string -%}
+ {%- set match.found = true -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif -%}
+ {%- if match.found -%}{%- set run.resolved_count = run.resolved_count + 1 -%}{%- endif -%}
+ {%- endfor -%}
+ {%- if run.tool_messages|length > 0 and run.resolved_count == run.tool_messages|length -%}
+ {%- set emitted = namespace(ids=[]) -%}
+ {%- for tool_call in state.tool_calls -%}
+ {%- if tool_call is mapping and tool_call.get('id') is not none and tool_call.get('id')|string not in emitted.ids -%}
+ {%- set emitted.ids = emitted.ids + [tool_call.get('id')|string] -%}
+ {%- set fn = tool_call.function if tool_call.function is defined and tool_call.function is mapping else tool_call -%}
+ {%- for tool_message in run.tool_messages -%}
+ {%- set result_id = tool_message.get('tool_call_id', tool_message.get('id')) -%}
+ {%- if result_id is not none and result_id|string == tool_call.get('id')|string -%}
+ {{- render_tool_message(tool_message, state, fn.get('name')) -}}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- else -%}
+ {%- for tool_message in run.tool_messages -%}
+ {{- render_tool_message(tool_message, state) -}}
+ {%- endfor -%}
+ {%- endif -%}
+ {%- endif -%}
+ {%- endif -%}
+{%- endfor -%}
+
+{%- if tool_choice is defined and tool_choice == 'required' -%}
+ {{- internal_system_message('tool-choice', 'The system is invoked with `tool_choice=required`.\nYou MUST call tools in the next message.') -}}
+{%- elif tool_choice is defined and tool_choice == 'none' -%}
+ {{- internal_system_message('tool-choice', 'The system is invoked with `tool_choice=none`.\nYou MUST NOT call any tools in the next message.') -}}
+{%- endif -%}
+
+{%- if response_schema is defined -%}
+ {%- set state.response_schema = response_schema -%}
+{%- elif response_format is defined and response_format is mapping and response_format.get('json_schema') is not none -%}
+ {%- set schema_wrapper = response_format.get('json_schema') -%}
+ {%- if schema_wrapper is mapping and 'schema' in schema_wrapper -%}
+ {%- set state.response_schema = schema_wrapper.get('schema') -%}
+ {%- elif schema_wrapper is mapping and 'json_schema' in schema_wrapper -%}
+ {%- set state.response_schema = schema_wrapper.get('json_schema') -%}
+ {%- else -%}
+ {%- set state.response_schema = schema_wrapper -%}
+ {%- endif -%}
+{%- endif -%}
+
+{%- set response_format_type = none -%}
+{%- if response_format is defined and response_format is mapping -%}
+ {%- set response_format_type = response_format.get('type') -%}
+{%- elif response_format is defined -%}
+ {%- set response_format_type = response_format -%}
+{%- endif -%}
+{%- if response_format_type == 'json_object' -%}
+ {{- internal_system_message(
+ 'response-format',
+ 'The system is invoked with `response_format=json_object`.\nYour response must be raw JSON data without markdown code blocks (```json) or any additional formatting.'
+ ) -}}
+{%- elif response_format_type == 'json_schema' -%}
+ {{- internal_system_message(
+ 'response-format',
+ 'The system is invoked with `response_format=json_schema`.\nYour response must be raw JSON data without markdown code blocks (```json) or any additional formatting.\nThe JSON data must match the following schema:\n```json\n' + json_sorted(state.response_schema) + '\n```'
+ ) -}}
+{%- endif -%}
+
+{%- if add_generation_prompt -%}
+ {{- open_tag('message', [('role', 'assistant')]) -}}
+ {{- open_tag('think' if thinking else 'response') -}}
+{%- endif -%}
+
+{%- if image_prompts is defined and image_prompts is not none and state.image_index != image_prompts|length -%}
+ {{- raise_exception('image prompt count ' + image_prompts|length|string + ' != consumed placeholder count ' + state.image_index|string) -}}
+{%- endif -%}
+
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 1cd9b758f3..c55f621cc6 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -144,6 +144,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_LLAMA_EMBED, "llama-embed" },
{ LLM_ARCH_MAINCODER, "maincoder" },
{ LLM_ARCH_KIMI_LINEAR, "kimi-linear" },
+ { LLM_ARCH_KIMI_K3, "kimi-k3" },
{ LLM_ARCH_TALKIE, "talkie" },
{ LLM_ARCH_MELLUM, "mellum" },
{ LLM_ARCH_NANBEIGE, "nanbeige" },
@@ -187,6 +188,9 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_FEATURES_LENGTH, "%s.features_length" },
{ LLM_KV_BLOCK_COUNT, "%s.block_count" },
{ LLM_KV_LEADING_DENSE_BLOCK_COUNT, "%s.leading_dense_block_count" },
+ { LLM_KV_ATTN_RES_BLOCK_SIZE, "%s.attn_res.block_size" },
+ { LLM_KV_ACTIVATION_SITU_BETA, "%s.activation.situ_beta" },
+ { LLM_KV_ACTIVATION_SITU_LINEAR_BETA, "%s.activation.situ_linear_beta" },
{ LLM_KV_FEED_FORWARD_LENGTH, "%s.feed_forward_length" },
{ LLM_KV_EXPERT_FEED_FORWARD_LENGTH, "%s.expert_feed_forward_length" },
{ LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, "%s.expert_shared_feed_forward_length" },
@@ -202,6 +206,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_EXPERT_GROUP_USED_COUNT, "%s.expert_group_used_count" },
{ LLM_KV_EXPERT_WEIGHTS_SCALE, "%s.expert_weights_scale" },
{ LLM_KV_EXPERT_WEIGHTS_NORM, "%s.expert_weights_norm" },
+ { LLM_KV_EXPERT_LATENT_LENGTH, "%s.expert_latent_length" },
{ LLM_KV_EXPERT_GATING_FUNC, "%s.expert_gating_func" },
{ LLM_KV_EXPERT_GROUP_SCALE, "%s.expert_group_scale" },
{ LLM_KV_EXPERTS_PER_GROUP, "%s.experts_per_group" },
@@ -313,6 +318,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" },
{ LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" },
+ { LLM_KV_KDA_GATE_LOWER_BOUND, "%s.kda.gate_lower_bound" },
{ LLM_KV_WKV_HEAD_SIZE, "%s.wkv.head_size" },
@@ -463,6 +469,13 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_SSM_F_B, "blk.%d.ssm_f_b" },
{ LLM_TENSOR_SSM_BETA, "blk.%d.ssm_beta" },
{ LLM_TENSOR_SSM_G_A, "blk.%d.ssm_g_a" },
+ { LLM_TENSOR_SSM_G, "blk.%d.ssm_g" },
+ { LLM_TENSOR_ATTN_RES_SCORE, "blk.%d.attn_res_score" },
+ { LLM_TENSOR_FFN_RES_SCORE, "blk.%d.ffn_res_score" },
+ { LLM_TENSOR_OUTPUT_RES_SCORE, "output_res_score" },
+ { LLM_TENSOR_FFN_ROUTED_DOWN, "blk.%d.ffn_routed_down" },
+ { LLM_TENSOR_FFN_ROUTED_UP, "blk.%d.ffn_routed_up" },
+ { LLM_TENSOR_FFN_ROUTED_NORM, "blk.%d.ffn_routed_norm" },
{ LLM_TENSOR_SSM_G_B, "blk.%d.ssm_g_b" },
{ LLM_TENSOR_SSM_NORM, "blk.%d.ssm_norm" },
{ LLM_TENSOR_ATTN_Q_A_NORM, "blk.%d.attn_q_a_norm" },
@@ -756,6 +769,13 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_SSM_F_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_SSM_BETA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_SSM_G_A, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_SSM_G, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_RES_SCORE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_RES_SCORE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_OUTPUT_RES_SCORE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_ROUTED_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_FFN_ROUTED_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_FFN_ROUTED_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_SSM_G_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_TIME_MIX_LERP_X, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_TIME_MIX_LN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
@@ -976,6 +996,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_QWEN3NEXT:
case LLM_ARCH_KIMI_LINEAR:
+ case LLM_ARCH_KIMI_K3:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_DEEPSEEK4:
@@ -1040,6 +1061,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_MINIMAX_M3:
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_KIMI_LINEAR:
+ case LLM_ARCH_KIMI_K3:
case LLM_ARCH_QWEN3TTS:
return false;
default:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 24252b9da9..9d74e3d7e0 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -145,6 +145,7 @@ enum llm_arch {
LLM_ARCH_LLAMA_EMBED,
LLM_ARCH_MAINCODER,
LLM_ARCH_KIMI_LINEAR,
+ LLM_ARCH_KIMI_K3,
LLM_ARCH_TALKIE,
LLM_ARCH_MELLUM,
LLM_ARCH_EAGLE3,
@@ -192,6 +193,9 @@ enum llm_kv {
LLM_KV_FEATURES_LENGTH,
LLM_KV_BLOCK_COUNT,
LLM_KV_LEADING_DENSE_BLOCK_COUNT,
+ LLM_KV_ATTN_RES_BLOCK_SIZE,
+ LLM_KV_ACTIVATION_SITU_BETA,
+ LLM_KV_ACTIVATION_SITU_LINEAR_BETA,
LLM_KV_FEED_FORWARD_LENGTH,
LLM_KV_EXPERT_FEED_FORWARD_LENGTH,
LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH,
@@ -207,6 +211,7 @@ enum llm_kv {
LLM_KV_EXPERT_GROUP_USED_COUNT,
LLM_KV_EXPERT_WEIGHTS_SCALE,
LLM_KV_EXPERT_WEIGHTS_NORM,
+ LLM_KV_EXPERT_LATENT_LENGTH,
LLM_KV_EXPERT_GATING_FUNC,
LLM_KV_EXPERT_GROUP_SCALE,
LLM_KV_EXPERTS_PER_GROUP,
@@ -318,6 +323,7 @@ enum llm_kv {
LLM_KV_SSM_DT_B_C_RMS,
LLM_KV_KDA_HEAD_DIM,
+ LLM_KV_KDA_GATE_LOWER_BOUND,
LLM_KV_WKV_HEAD_SIZE,
@@ -492,6 +498,13 @@ enum llm_tensor {
LLM_TENSOR_SSM_BETA, // kimi: beta mixing coefficient and qwen3.5
LLM_TENSOR_SSM_G_A, // kimi: output gate projection A
LLM_TENSOR_SSM_G_B, // kimi: output gate projection B
+ LLM_TENSOR_SSM_G, // kimi-k3: full-rank KDA gate
+ LLM_TENSOR_ATTN_RES_SCORE, // kimi-k3: fused res_norm*res_proj (pre-attn)
+ LLM_TENSOR_FFN_RES_SCORE, // kimi-k3: fused res_norm*res_proj (pre-ffn)
+ LLM_TENSOR_OUTPUT_RES_SCORE, // kimi-k3: fused res_norm*res_proj (final)
+ LLM_TENSOR_FFN_ROUTED_DOWN, // kimi-k3: latent MoE down
+ LLM_TENSOR_FFN_ROUTED_UP, // kimi-k3: latent MoE up
+ LLM_TENSOR_FFN_ROUTED_NORM, // kimi-k3: latent MoE norm
LLM_TENSOR_TIME_MIX_W0,
LLM_TENSOR_TIME_MIX_W1,
LLM_TENSOR_TIME_MIX_W2,
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index ce38e8a314..792bb6cac8 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -2293,7 +2293,10 @@ void llama_context::output_reorder() {
uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
uint32_t res;
- if (model.arch == LLM_ARCH_QWEN3NEXT ||
+ if (model.arch == LLM_ARCH_KIMI_K3) {
+ // the n_tokens*40 budget below is exhausted at ubatch 3840
+ res = std::max<uint32_t>(n_tokens * 160, 64u * model.n_tensors());
+ } else if (model.arch == LLM_ARCH_QWEN3NEXT ||
model.arch == LLM_ARCH_KIMI_LINEAR ||
model.arch == LLM_ARCH_QWEN35 ||
model.arch == LLM_ARCH_QWEN35MOE ||
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 55d8580246..1896758c5d 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -1835,6 +1835,8 @@ ggml_tensor * llm_graph_context::build_ffn(
cur = ggml_reglu(ctx0, cur);
cb(cur, "ffn_reglu", il);
} break;
+ case LLM_FFN_SITU:
+ GGML_ABORT("not yet supported");
default:
GGML_ABORT("fatal error");
}
@@ -2174,6 +2176,21 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
cur = ggml_silu(ctx0, cur);
cb(cur, "ffn_moe_silu", il);
} break;
+ case LLM_FFN_SITU:
+ {
+ // situ(gate, up) = beta*tanh(gate/beta)*sigmoid(gate) * lb*tanh(up/lb)
+ GGML_ASSERT(has_gate);
+ const float beta = hparams.situ_beta;
+ const float lb = hparams.situ_linear_beta;
+
+ ggml_tensor * act = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/beta)), beta);
+ act = ggml_mul(ctx0, act, ggml_sigmoid(ctx0, cur));
+ if (lb > 0.0f) {
+ up = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, up, 1.0f/lb)), lb);
+ }
+ cur = ggml_mul(ctx0, act, up);
+ cb(cur, "ffn_moe_situ", il);
+ } break;
case LLM_FFN_GELU:
if (has_gate) {
cur = ggml_geglu_split(ctx0, cur, up);
diff --git a/src/llama-graph.h b/src/llama-graph.h
index 75bc0fe80d..94324c7457 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -59,6 +59,7 @@ enum llm_ffn_op_type : int {
LLM_FFN_GEGLU,
LLM_FFN_REGLU,
LLM_FFN_SWIGLU_OAI_MOE,
+ LLM_FFN_SITU, // kimi-k3
};
enum llm_ffn_gate_type {
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index d9ac112b79..06b7102250 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -4,10 +4,11 @@
#include <array>
#include <cassert>
+#include <cmath>
// bump if necessary
#define LLAMA_MAX_LAYERS 512
-#define LLAMA_MAX_EXPERTS 512 // Qwen3 Next
+#define LLAMA_MAX_EXPERTS 1024 // Kimi K3
enum llama_expert_gating_func_type {
LLAMA_EXPERT_GATING_FUNC_TYPE_NONE = 0,
@@ -170,6 +171,13 @@ struct llama_hparams {
// for Kimi Linear KDA
uint32_t n_embd_head_kda = 0;
+ // kimi-k3
+ uint32_t n_expert_latent = 0; // routed_expert_hidden_size (0 = experts run at n_embd)
+ uint32_t attn_res_block_size = 0; // 0 = no cross-layer attention residuals
+ float kda_gate_lower_bound = -INFINITY;
+ float situ_beta = 1.0f;
+ float situ_linear_beta = 0.0f; // 0 = no linear-beta transform on the up branch
+
bool ssm_dt_b_c_rms = false;
float f_clamp_kqv = 0.0f;
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index abca773a9a..2e2e2d3d28 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -213,6 +213,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true);
add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
+ add_kv(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent);
add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
add_kv(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);
add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp);
@@ -319,6 +320,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms);
add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);
+ add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound);
add_kv(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size);
@@ -376,6 +378,10 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_XIELU_BETA, hparams.xielu_beta);
add_kv(LLM_KV_XIELU_EPS, hparams.xielu_eps);
+ add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
+ add_kv(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
+ add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);
+
// deprecated
// add_kv(LLM_KV_TOKENIZER_PREFIX_ID, ???);
// add_kv(LLM_KV_TOKENIZER_SUFFIX_ID, ???);
@@ -403,6 +409,7 @@ void llama_model_saver::add_tensors_from_model() {
add_tensor(model->output_norm_enc);
add_tensor(model->output_s);
add_tensor(model->output_in_s);
+ add_tensor(model->output_res_score);
add_tensor(model->cls);
add_tensor(model->cls_b);
add_tensor(model->cls_out);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 3ef683d5be..25df27318d 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -322,6 +322,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_mimo2(params);
case LLM_ARCH_KIMI_LINEAR:
return new llama_model_kimi_linear(params);
+ case LLM_ARCH_KIMI_K3:
+ return new llama_model_kimi_k3(params);
case LLM_ARCH_STEP35:
return new llama_model_step35(params);
default:
@@ -845,6 +847,7 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_397B_A17B: return "397B.A17B";
case LLM_TYPE_685B_A37B: return "685B.A37B";
case LLM_TYPE_744B_A40B: return "744B.A40B";
+ case LLM_TYPE_2_8T_A50B: return "2.8T.A50B";
case LLM_TYPE_E2B: return "E2B";
case LLM_TYPE_E4B: return "E4B";
default: return "?B";
@@ -2602,6 +2605,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_KIMI_LINEAR:
+ case LLM_ARCH_KIMI_K3:
return LLAMA_ROPE_TYPE_NONE;
// use what we call a normal RoPE, operating on pairs of consecutive head values
diff --git a/src/llama-model.h b/src/llama-model.h
index 510ed8498b..315c6d7b4e 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -144,6 +144,7 @@ enum llm_type {
LLM_TYPE_397B_A17B, // Qwen3.5
LLM_TYPE_685B_A37B, // DeepSeek V3.2
LLM_TYPE_744B_A40B, // GLM-5
+ LLM_TYPE_2_8T_A50B, // Kimi-K3
LLM_TYPE_E2B,
LLM_TYPE_E4B,
};
@@ -530,6 +531,14 @@ struct llama_layer {
struct ggml_tensor * ssm_g_b = nullptr;
struct ggml_tensor * ssm_o_norm = nullptr;
+ // kimi-k3
+ struct ggml_tensor * ssm_g = nullptr; // full-rank KDA gate (replaces ssm_g_a/ssm_g_b)
+ struct ggml_tensor * attn_res_score = nullptr; // fused res_norm*res_proj, pre-attention
+ struct ggml_tensor * ffn_res_score = nullptr; // fused res_norm*res_proj, pre-FFN
+ struct ggml_tensor * ffn_routed_down = nullptr; // latent MoE: n_embd -> n_expert_latent
+ struct ggml_tensor * ffn_routed_up = nullptr; // latent MoE: n_expert_latent -> n_embd
+ struct ggml_tensor * ffn_routed_norm = nullptr;
+
// DSA (deepseek sparse attention)
struct ggml_tensor * indexer_k_norm = nullptr;
struct ggml_tensor * indexer_k_norm_b = nullptr;
@@ -589,6 +598,7 @@ struct llama_model {
struct ggml_tensor * tok_norm_b = nullptr;
struct ggml_tensor * output_norm = nullptr;
+ struct ggml_tensor * output_res_score = nullptr; // kimi-k3: final cross-layer residual mix
struct ggml_tensor * output_norm_b = nullptr;
struct ggml_tensor * output = nullptr;
struct ggml_tensor * output_b = nullptr;
diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp
new file mode 100644
index 0000000000..d952d72cdf
--- /dev/null
+++ b/src/models/kimi-k3.cpp
@@ -0,0 +1,614 @@
+#include "models.h"
+#include "llama-memory-recurrent.h"
+
+//
+// Kimi-K3 text model: hybrid KDA (linear) + MLA (full) attention, as in kimi-linear.
+// Parts that kimi-linear does not have:
+// 1. cross-layer residual attention (attn_res_block_size)
+// 2. latent MoE (routed experts run at n_expert_latent)
+// 3. situ activation (replaces SwiGLU everywhere)
+// 4. MLA output gate (sigmoid gate before o_proj)
+// 5. full-rank KDA gate (single ssm_g instead of ssm_g_a/ssm_g_b)
+//
+
+void llama_model_kimi_k3::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl);
+ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);
+ ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q, false);
+ ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);
+ ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
+ ml.get_key(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);
+ ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound, false);
+
+ // the MLA cache holds the compressed latent
+ // set it here too, as older GGUFs have no value_length key
+ hparams.n_embd_head_v_full = hparams.n_lora_kv;
+
+ // n_head_kv == 0 marks a KDA (recurrent) layer, as in kimi-linear
+ for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
+ hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;
+ }
+
+ ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
+ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
+ ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
+ ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
+ ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent, false);
+
+ ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
+ ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
+ ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);
+
+ switch (hparams.n_layer()) {
+ case 93: type = LLM_TYPE_2_8T_A50B; break; // Kimi-K3
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_kimi_k3::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ const int64_t n_embd_latent = hparams.n_expert_latent > 0 ? hparams.n_expert_latent : n_embd;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);
+
+ if (hparams.attn_res_block_size > 0) {
+ output_res_score = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_SCORE, "weight"), {n_embd}, 0);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+ if (hparams.attn_res_block_size > 0) {
+ layer.attn_res_score = create_tensor(tn(LLM_TENSOR_ATTN_RES_SCORE, "weight", i), {n_embd}, 0);
+ layer.ffn_res_score = create_tensor(tn(LLM_TENSOR_FFN_RES_SCORE, "weight", i), {n_embd}, 0);
+ }
+
+ const int64_t head_dim = hparams.n_embd_head_kda;
+ const int64_t d_conv = hparams.ssm_d_conv;
+ const int64_t d_inner = head_dim * n_head;
+
+ if (hparams.is_recr(i)) {
+ // conv1d may be stored 4D [d_conv, 1, d_inner, 1] or 3D (quantization drops the trailing 1)
+ auto conv = [&](llm_tensor tid) {
+ ggml_tensor * t = create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);
+ return t ? t : create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner}, 0);
+ };
+ layer.ssm_q_conv = conv(LLM_TENSOR_SSM_CONV1D_Q);
+ layer.ssm_k_conv = conv(LLM_TENSOR_SSM_CONV1D_K);
+ layer.ssm_v_conv = conv(LLM_TENSOR_SSM_CONV1D_V);
+
+ create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, 0);
+
+ layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim}, 0);
+ layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim, d_inner}, 0);
+ layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);
+
+ // K3's A_log is a plain 1-D [n_head] tensor (kimi-linear's is padded)
+ layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {n_head}, 0);
+ layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0);
+
+ // K3 uses a single full-rank gate instead of kimi-linear's g_a/g_b pair
+ layer.ssm_g = create_tensor(tn(LLM_TENSOR_SSM_G, "weight", i), {n_embd, d_inner}, 0);
+ layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0);
+ } else {
+ const int64_t q_lora_rank = hparams.n_lora_q;
+ const int64_t kv_lora_rank = hparams.n_lora_kv;
+ const int64_t n_embd_head_k = hparams.n_embd_head_k_mla();
+ const int64_t n_embd_head_v = hparams.n_embd_head_v_mla();
+ const int64_t qk_rope_head_dim = hparams.n_rot();
+ const int64_t qk_nope_head_dim = n_embd_head_k - qk_rope_head_dim;
+
+ layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, TENSOR_NOT_REQUIRED);
+ layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0);
+
+ if (layer.attn_q_a_norm) {
+ layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0);
+ layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k}, 0);
+ } else {
+ layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k}, 0);
+ }
+
+ layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + qk_rope_head_dim}, 0);
+ layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i),
+ {kv_lora_rank, n_head * (qk_nope_head_dim + n_embd_head_v)},
+ TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
+ if (!layer.wkv_b) {
+ layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {qk_nope_head_dim, kv_lora_rank, n_head}, 0);
+ layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v, n_head}, 0);
+ }
+
+ // K3: sigmoid output gate applied to the attention output before o_proj
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head * n_embd_head_v}, TENSOR_NOT_REQUIRED);
+
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v, n_embd}, 0);
+ }
+
+ if (i < (int) hparams.n_layer_dense_lead) {
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+ } else {
+ const int64_t n_ff_exp = hparams.n_ff_exp;
+
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
+
+ // routed experts live in the latent space
+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd_latent, n_expert}, 0);
+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0);
+
+ if (hparams.n_expert_latent > 0) {
+ layer.ffn_routed_down = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_DOWN, "weight", i), {n_embd, n_embd_latent}, 0);
+ layer.ffn_routed_up = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_UP, "weight", i), {n_embd_latent, n_embd}, 0);
+ layer.ffn_routed_norm = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_NORM, "weight", i), {n_embd_latent}, TENSOR_NOT_REQUIRED);
+ }
+
+ // shared experts stay at n_embd, width = moe_intermediate_size * n_expert_shared
+ const int64_t n_ff_shexp = n_ff_exp * (hparams.n_expert_shared > 0 ? hparams.n_expert_shared : 1);
+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED);
+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, TENSOR_NOT_REQUIRED);
+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, TENSOR_NOT_REQUIRED);
+ }
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_kimi_k3::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+// situ(gate, up) = beta*tanh(gate/beta)*sigmoid(gate) * linear_beta*tanh(up/linear_beta)
+// linear_beta <= 0 disables the transform on the up branch
+static ggml_tensor * kimi_k3_situ(ggml_context * ctx0, ggml_tensor * gate, ggml_tensor * up,
+ float beta, float linear_beta) {
+ ggml_tensor * a = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, gate, 1.0f/beta)), beta);
+ a = ggml_mul(ctx0, a, ggml_sigmoid(ctx0, gate));
+
+ if (linear_beta > 0.0f) {
+ up = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, up, 1.0f/linear_beta)), linear_beta);
+ }
+ return ggml_mul(ctx0, a, up);
+}
+
+//
+// cross-layer residual attention
+//
+
+// layout is [n_embd, n_ckpt, n_tokens]: rms_norm reduces over ne0, dsv4_hc_pre over ne1
+// append the new checkpoint, do not re-fold the whole chain
+void llama_model_kimi_k3::graph::res_push(ggml_tensor * cur, int64_t n_embd, int64_t n_tokens) {
+ ggml_tensor * ckpt = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
+
+ resi_stack = resi_stack ? ggml_concat(ctx0, resi_stack, ckpt, 1) : ckpt;
+}
+
+ggml_tensor * llama_model_kimi_k3::graph::res_mix(ggml_tensor * cur, ggml_tensor * score_w,
+ int64_t n_tokens, int il) {
+ if (!resi_stack) {
+ return cur; // layer 0: nothing banked yet
+ }
+
+ const int n_ckpt = (int) resi_stack->ne[1];
+ const float eps = hparams.f_norm_rms_eps;
+
+ ggml_tensor * src = resi_stack; // [n_embd, n_ckpt, n_tokens]
+
+ // one rms_norm scores all checkpoints at once
+ // note: the scores use the normalized values, but the sum below uses the raw ones
+ ggml_tensor * sc_src = ggml_rms_norm(ctx0, src, eps);
+ sc_src = ggml_mul(ctx0, sc_src, score_w);
+ sc_src = ggml_sum_rows(ctx0, sc_src); // [1, n_ckpt, n_tokens]
+ sc_src = ggml_reshape_2d(ctx0, sc_src, n_ckpt, n_tokens);
+
+ // the current residual stream is scored apart, so the stack stays append-only
+ ggml_tensor * sc_cur = ggml_rms_norm(ctx0, cur, eps);
+ sc_cur = ggml_mul(ctx0, sc_cur, score_w);
+ sc_cur = ggml_sum_rows(ctx0, sc_cur); // [1, n_tokens]
+
+ ggml_tensor * scores = ggml_concat(ctx0, sc_src, sc_cur, 0); // [n_ckpt+1, n_tokens]
+ ggml_tensor * probs = ggml_soft_max(ctx0, scores); // over ne0 = n_ckpt+1
+ cb(probs, "res_probs", il);
+
+ // split the sum: hc_pre handles the stack, a broadcast-multiply the current stream
+ ggml_tensor * p_src = ggml_cont(ctx0, ggml_view_2d(ctx0, probs, n_ckpt, n_tokens, probs->nb[1], 0));
+ ggml_tensor * p_cur = ggml_cont(ctx0, ggml_view_2d(ctx0, probs, 1, n_tokens, probs->nb[1],
+ probs->nb[0] * n_ckpt));
+
+ ggml_tensor * out = ggml_dsv4_hc_pre(ctx0, src, p_src);
+ out = ggml_add(ctx0, out, ggml_mul(ctx0, cur, p_cur));
+
+ return out;
+}
+
+llama_model_kimi_k3::graph::graph(const llama_model & model, const llm_graph_params & params) :
+ llm_build_delta_net_base(params), model(model) {
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ inpL = build_inp_embd(model.tok_embd);
+ cb(inpL, "inp_embd", -1);
+
+ // K3 MLA is nope-only, so there is no position input
+
+ auto * inp_kv = !hparams.is_mla() ? build_inp_mem_hybrid() : nullptr;
+ auto * inp_k = hparams.is_mla() ? build_inp_mem_hybrid_k() : nullptr;
+ auto * inp_rs = hparams.is_mla() ? inp_k->get_recr() : inp_kv->get_recr();
+ auto * inp_attn_kv = !hparams.is_mla() ? inp_kv->get_attn() : nullptr;
+ auto * inp_attn_k = hparams.is_mla() ? inp_k->get_attn() : nullptr;
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ const int64_t n_head_kda = hparams.n_head();
+ const int64_t head_dim = hparams.n_embd_head_kda;
+ const int64_t d_conv = hparams.ssm_d_conv;
+ const int64_t d_inner = n_head_kda * head_dim;
+ const int64_t n_seqs = ubatch.n_seqs;
+ const int64_t n_seq_tokens = ubatch.n_seq_tokens;
+
+ GGML_ASSERT(n_seqs != 0);
+ GGML_ASSERT(ubatch.equal_seqs());
+ GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
+
+ const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();
+ const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();
+ const int64_t kv_lora_rank = hparams.n_lora_kv;
+ const int64_t n_embd_head_qk_rope = hparams.n_rot();
+ const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
+ const float kq_scale_mla = 1.0f / sqrtf((float) n_embd_head_k_mla);
+
+ const uint32_t res_bs = hparams.attn_res_block_size;
+ const bool use_attn_res = res_bs > 0;
+ const int64_t n_embd_latent = hparams.n_expert_latent > 0 ? hparams.n_expert_latent : n_embd;
+
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+
+ // the residual stream, banked on checkpoint layers and then restarted
+ // from the attention output alone
+ ggml_tensor * prefix_sum = inpL;
+
+ cur = use_attn_res ? res_mix(prefix_sum, layer.attn_res_score, n_tokens, il)
+ : prefix_sum;
+
+ bool banked = false;
+ if (use_attn_res && (uint32_t) il % res_bs == 0) {
+ res_push(prefix_sum, n_embd, n_tokens); // banks the RAW layer input, not `cur`
+ banked = true;
+ }
+
+ cur = build_norm(cur, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+ ggml_build_forward_expand(gf, cur);
+
+ if (hparams.is_recr(il)) {
+ cur = build_kda_layer(cur, layer, inp_rs, d_conv, head_dim, n_head_kda,
+ d_inner, n_seq_tokens, n_seqs, il);
+ } else {
+ cur = build_mla_layer(cur, layer, inp_attn_k, inp_attn_kv,
+ n_embd_head_k_mla, n_embd_head_v_mla, kv_lora_rank,
+ n_embd_head_qk_rope, n_embd_head_qk_nope, kq_scale_mla, il);
+ }
+
+ prefix_sum = banked ? cur : ggml_add(ctx0, prefix_sum, cur);
+ cb(prefix_sum, "prefix_sum_attn", il);
+
+ cur = use_attn_res ? res_mix(prefix_sum, layer.ffn_res_score, n_tokens, il)
+ : prefix_sum;
+
+ cur = build_norm(cur, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ if ((uint32_t) il < hparams.n_layer_dense_lead) {
+ ggml_tensor * g = ggml_mul_mat(ctx0, layer.ffn_gate, cur);
+ ggml_tensor * u = ggml_mul_mat(ctx0, layer.ffn_up, cur);
+ cur = kimi_k3_situ(ctx0, g, u, hparams.situ_beta, hparams.situ_linear_beta);
+ cur = ggml_mul_mat(ctx0, layer.ffn_down, cur);
+ cb(cur, "ffn_out", il);
+ } else {
+ cur = build_latent_moe(cur, layer, n_embd_latent, il);
+ }
+
+ prefix_sum = ggml_add(ctx0, prefix_sum, cur);
+ prefix_sum = build_cvec(prefix_sum, il);
+ cb(prefix_sum, "l_out", il);
+
+ inpL = prefix_sum;
+ }
+
+ cur = inpL;
+
+ // final mix, then narrow to the output tokens
+ if (use_attn_res) {
+ cur = res_mix(cur, model.output_res_score, n_tokens, -1);
+ }
+ if (inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ }
+
+ cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = ggml_mul_mat(ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
+
+//
+// KDA layer
+//
+
+// causal conv1d over one of Q/K/V. `qkv` selects which third of the conv state to use
+static ggml_tensor * kimi_k3_conv1d(ggml_cgraph * gf, ggml_context * ctx0,
+ ggml_tensor * conv_states_all, ggml_tensor * conv_state_all,
+ int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w,
+ int64_t d_conv, int64_t head_dim, int64_t n_head,
+ int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head) {
+ const int64_t d_inner = head_dim * n_head;
+ const int64_t conv_state_size = (d_conv - 1) * d_inner;
+ const int64_t n_embd_r_total = 3 * conv_state_size;
+
+ ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs,
+ (d_conv - 1) * ggml_element_size(conv_state_all),
+ n_embd_r_total * ggml_element_size(conv_state_all),
+ qkv * conv_state_size * ggml_element_size(conv_state_all));
+
+ ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x);
+ ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs);
+ ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0);
+
+ ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,
+ conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]);
+ ggml_build_forward_expand(gf,
+ ggml_cpy(ctx0, last_conv_x,
+ ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs,
+ (d_conv - 1) * ggml_element_size(conv_states_all),
+ n_embd_r_total * ggml_element_size(conv_states_all),
+ (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all))));
+
+ ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner);
+ ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight);
+ Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens);
+ Xcur = ggml_silu(ctx0, Xcur);
+
+ return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs);
+}
+
+ggml_tensor * llama_model_kimi_k3::graph::build_kda_layer(
+ ggml_tensor * cur, const llama_layer & layer, llm_graph_input_rs * inp_rs,
+ int64_t d_conv, int64_t head_dim, int64_t n_head_kda,
+ int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il) {
+
+ const auto * mctx_cur = inp_rs->mctx;
+ const auto kv_head = mctx_cur->get_head();
+
+ ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
+ ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);
+
+ ggml_tensor * Qcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head);
+ ggml_tensor * Kcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head);
+ ggml_tensor * Vcur = kimi_k3_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head_kda, n_seq_tokens, n_seqs, n_tokens, kv_head);
+ cb(Qcur, "kda_q_conv", il);
+ cb(Kcur, "kda_k_conv", il);
+ cb(Vcur, "kda_v_conv", il);
+
+ // gate_lower_bound is not a clamp - when set, it swaps the decay gate activation:
+ // unset (kimi-linear): g = -exp(A_log) * softplus(f_b(f_a(x)) + dt_bias)
+ // set (K3, -5.0): g = lower_bound * sigmoid(exp(A_log) * (f_b(f_a(x)) + dt_bias))
+ // ssm_a holds -exp(A_log) (folded at conversion time), so exp(A_log) == -ssm_a
+ ggml_tensor * f_a = ggml_mul_mat(ctx0, layer.ssm_f_a, cur);
+ ggml_tensor * g1 = ggml_mul_mat(ctx0, layer.ssm_f_b, f_a);
+ g1 = ggml_add(ctx0, g1, layer.ssm_dt_b);
+
+ ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head_kda, 1);
+
+ if (hparams.kda_gate_lower_bound > -INFINITY) {
+ g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head_kda, n_tokens);
+ g1 = ggml_mul(ctx0, g1, A); // -exp(A_log) * (...)
+ g1 = ggml_sigmoid(ctx0, ggml_scale(ctx0, g1, -1.0f));
+ g1 = ggml_scale(ctx0, g1, hparams.kda_gate_lower_bound);
+ } else {
+ g1 = ggml_softplus(ctx0, g1);
+ g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head_kda, n_tokens);
+ g1 = ggml_mul(ctx0, g1, A);
+ }
+ cb(g1, "kda_g1", il);
+
+ g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head_kda, n_seq_tokens, n_seqs);
+
+ ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur);
+ beta = ggml_reshape_4d(ctx0, beta, 1, n_head_kda, n_seq_tokens, n_seqs);
+ beta = ggml_sigmoid(ctx0, beta);
+ cb(beta, "kda_beta", il);
+
+ ggml_tensor * cur_3d = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);
+
+ ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);
+ ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs);
+ state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head_kda, n_seqs);
+
+ const float eps = hparams.f_norm_rms_eps;
+ Qcur = ggml_l2_norm(ctx0, Qcur, eps);
+ Kcur = ggml_l2_norm(ctx0, Kcur, eps);
+
+ auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il);
+
+ ggml_tensor * output = ggml_cont(ctx0, attn_out.first);
+ cb(output, "kda_scan_out", il);
+ ggml_tensor * new_state = attn_out.second;
+
+ ggml_build_forward_expand(gf,
+ ggml_cpy(ctx0, new_state,
+ ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,
+ kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));
+
+ // K3: single full-rank gate (kimi-linear factors this as g_b(g_a(x)))
+ ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur_3d, cur_3d->ne[0], n_seq_tokens * n_seqs);
+ ggml_tensor * g2 = ggml_mul_mat(ctx0, layer.ssm_g, cur_2d);
+ g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head_kda, n_seq_tokens * n_seqs);
+
+ ggml_tensor * o = ggml_reshape_3d(ctx0, output, head_dim, n_head_kda, n_seq_tokens * n_seqs);
+ ggml_tensor * normed = build_norm(o, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il);
+ cb(g2, "kda_g2", il);
+ cb(normed, "kda_normed", il);
+ ggml_tensor * gated = ggml_mul(ctx0, normed, ggml_sigmoid(ctx0, g2));
+
+ gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens);
+ cur = ggml_mul_mat(ctx0, layer.wo, gated);
+ cb(cur, "kda_out", il);
+
+ return cur;
+}
+
+//
+// MLA layer (nope-only, with K3's sigmoid output gate)
+//
+
+ggml_tensor * llama_model_kimi_k3::graph::build_mla_layer(
+ ggml_tensor * cur, const llama_layer & layer,
+ llm_graph_input_attn_k * inp_attn_k, llm_graph_input_attn_kv * inp_attn_kv,
+ int64_t n_embd_head_k_mla, int64_t n_embd_head_v_mla, int64_t kv_lora_rank,
+ int64_t n_embd_head_qk_rope, int64_t n_embd_head_qk_nope, float kq_scale, int il) {
+
+ ggml_tensor * inp_gate = cur; // the output gate reads the *normed* layer input
+
+ ggml_tensor * Qcur;
+ if (layer.wq_a) {
+ Qcur = ggml_mul_mat(ctx0, layer.wq_a, cur);
+ Qcur = build_norm(Qcur, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
+ Qcur = ggml_mul_mat(ctx0, layer.wq_b, Qcur);
+ } else {
+ Qcur = ggml_mul_mat(ctx0, layer.wq, cur);
+ }
+
+ ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur);
+
+ ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);
+ ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
+
+ // no RoPE: mla_use_nope is asserted at conversion time
+ kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
+
+ ggml_tensor * out;
+ if (layer.wk_b && layer.wv_b) {
+ ggml_tensor * q_nope = ggml_view_3d(ctx0, Qcur, n_embd_head_qk_nope, n_head, n_tokens,
+ ggml_row_size(Qcur->type, n_embd_head_k_mla),
+ ggml_row_size(Qcur->type, n_embd_head_k_mla) * n_head, 0);
+ ggml_tensor * q_pe = ggml_view_3d(ctx0, Qcur, n_embd_head_qk_rope, n_head, n_tokens,
+ ggml_row_size(Qcur->type, n_embd_head_k_mla),
+ ggml_row_size(Qcur->type, n_embd_head_k_mla) * n_head,
+ ggml_row_size(Qcur->type, n_embd_head_qk_nope));
+
+ q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
+ ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);
+ q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);
+
+ ggml_tensor * Q = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
+ ggml_tensor * kv_cmpr_3d = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);
+ ggml_tensor * K = ggml_concat(ctx0, kv_cmpr_3d, k_pe, 0);
+ ggml_tensor * V = kv_cmpr_3d;
+
+ // wo == NULL: the output projection is applied after the gate below
+ out = build_attn(inp_attn_k, nullptr, NULL, nullptr, Q, K, V, nullptr, nullptr, layer.wv_b, kq_scale, il);
+ } else {
+ ggml_tensor * Q = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k_mla, n_head, n_tokens);
+ ggml_tensor * kv = ggml_mul_mat(ctx0, layer.wkv_b, kv_cmpr);
+ const int64_t kv_per_head = n_embd_head_qk_nope + n_embd_head_v_mla;
+
+ ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,
+ ggml_row_size(kv->type, kv_per_head), ggml_row_size(kv->type, kv_per_head * n_head), 0);
+ ggml_tensor * V = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd_head_v_mla, n_head, n_tokens,
+ ggml_row_size(kv->type, kv_per_head), ggml_row_size(kv->type, kv_per_head * n_head),
+ ggml_row_size(kv->type, n_embd_head_qk_nope)));
+
+ ggml_tensor * k_pe_t = ggml_new_tensor_3d(ctx0, k_pe->type, n_embd_head_qk_rope, n_head, n_tokens);
+ ggml_tensor * K = ggml_concat(ctx0, ggml_repeat(ctx0, k_pe, k_pe_t), k_nope, 0);
+
+ out = build_attn(inp_attn_kv, nullptr, NULL, nullptr, Q, K, V, nullptr, nullptr, nullptr, kq_scale, il);
+ }
+
+ // K3: attn_output *= sigmoid(g_proj(x)), then o_proj
+ if (layer.wqkv_gate) {
+ ggml_tensor * g = ggml_sigmoid(ctx0, ggml_mul_mat(ctx0, layer.wqkv_gate, inp_gate));
+ out = ggml_mul(ctx0, out, g);
+ cb(out, "mla_gated", il);
+ }
+
+ out = ggml_mul_mat(ctx0, layer.wo, out);
+ cb(out, "mla_out", il);
+
+ return out;
+}
+
+//
+// latent MoE: down-project, run the routed experts in the latent space, norm, up-project;
+// shared experts stay at n_embd and read the un-projected input.
+//
+
+ggml_tensor * llama_model_kimi_k3::graph::build_latent_moe(
+ ggml_tensor * cur, const llama_layer & layer, int64_t n_embd_latent, int il) {
+
+ ggml_tensor * identity = cur;
+
+ ggml_tensor * routed_in = layer.ffn_routed_down
+ ? ggml_mul_mat(ctx0, layer.ffn_routed_down, cur)
+ : cur;
+
+ // the router scores the full-width input while the experts take the latent one,
+ // so the logits are computed here and passed to build_moe_ffn
+ ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, identity);
+ cb(logits, "ffn_moe_logits", il);
+
+ ggml_tensor * moe_out = build_moe_ffn(routed_in,
+ nullptr, // gate_inp unused: the logits above are passed instead
+ layer.ffn_up_exps,
+ layer.ffn_gate_exps,
+ layer.ffn_down_exps,
+ layer.ffn_exp_probs_b,
+ hparams.n_expert,
+ hparams.n_expert_used,
+ LLM_FFN_SITU, hparams.expert_weights_norm,
+ hparams.expert_weights_scale,
+ (llama_expert_gating_func_type) hparams.expert_gating_func,
+ il,
+ logits);
+ cb(moe_out, "ffn_moe_out", il);
+
+ if (layer.ffn_routed_norm) {
+ moe_out = build_norm(moe_out, layer.ffn_routed_norm, NULL, LLM_NORM_RMS, il);
+ }
+ if (layer.ffn_routed_up) {
+ moe_out = ggml_mul_mat(ctx0, layer.ffn_routed_up, moe_out);
+ }
+ GGML_UNUSED(n_embd_latent);
+
+ if (layer.ffn_gate_shexp) {
+ ggml_tensor * g = ggml_mul_mat(ctx0, layer.ffn_gate_shexp, identity);
+ ggml_tensor * u = ggml_mul_mat(ctx0, layer.ffn_up_shexp, identity);
+ ggml_tensor * sh = kimi_k3_situ(ctx0, g, u, hparams.situ_beta, hparams.situ_linear_beta);
+ sh = ggml_mul_mat(ctx0, layer.ffn_down_shexp, sh);
+ cb(sh, "ffn_shexp", il);
+ moe_out = ggml_add(ctx0, moe_out, sh);
+ }
+
+ cb(moe_out, "ffn_out", il);
+ return moe_out;
+}
diff --git a/src/models/models.h b/src/models/models.h
index cefaf56eb1..a181370474 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2285,6 +2285,42 @@ struct llama_model_mimo2 : public llama_model_base {
};
+struct llama_model_kimi_k3 : public llama_model_base {
+ llama_model_kimi_k3(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_build_delta_net_base {
+ graph(const llama_model & model, const llm_graph_params & params);
+
+ const llama_model & model;
+
+ // Cross-layer residual attention (K3's `_apply_attn_res`).
+ ggml_tensor * resi_stack = nullptr;
+
+ void res_push(ggml_tensor * cur, int64_t n_embd, int64_t n_tokens);
+ ggml_tensor * res_mix(ggml_tensor * cur, ggml_tensor * score_w,
+ int64_t n_tokens, int il);
+
+ ggml_tensor * build_kda_layer(ggml_tensor * cur, const llama_layer & layer,
+ llm_graph_input_rs * inp_rs,
+ int64_t d_conv, int64_t head_dim, int64_t n_head_kda,
+ int64_t d_inner, int64_t n_seq_tokens, int64_t n_seqs, int il);
+
+ ggml_tensor * build_mla_layer(ggml_tensor * cur, const llama_layer & layer,
+ llm_graph_input_attn_k * inp_attn_k,
+ llm_graph_input_attn_kv * inp_attn_kv,
+ int64_t n_embd_head_k_mla, int64_t n_embd_head_v_mla,
+ int64_t kv_lora_rank, int64_t n_embd_head_qk_rope,
+ int64_t n_embd_head_qk_nope, float kq_scale, int il);
+
+ ggml_tensor * build_latent_moe(ggml_tensor * cur, const llama_layer & layer,
+ int64_t n_embd_latent, int il);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
struct llama_model_kimi_linear : public llama_model_base {
llama_model_kimi_linear(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index 51ad854849..c4670da853 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -4462,6 +4462,109 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
}
}
+ // Kimi-K3 tests - custom parser
+ // Unique feature: XTML tags built from <|open|>/<|close|>/<|sep|>, and a
+ // generation prompt that leaves the think section already open.
+ {
+ auto tst = peg_tester("models/templates/Kimi-K3.jinja", detailed_debug);
+
+ // Content only. The response section is explicit even with no reasoning.
+ tst.test("<|open|>response<|sep|>Hello, world!\nWhat's up?<|close|>response<|sep|>"
+ "<|close|>message<|sep|>")
+ .expect(message_assist)
+ .run();
+
+ // Reasoning with no opening tag - the generation prompt already opened it
+ tst.test("I'm thinking about this<|close|>think<|sep|>"
+ "<|open|>response<|sep|>Hello, world!\nWhat's up?<|close|>response<|sep|>"
+ "<|close|>message<|sep|>")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .expect(simple_assist_msg("Hello, world!\nWhat's up?", "I'm thinking about this"))
+ .run();
+
+ // Prose that mentions the tag names must survive intact.
+ tst.test("<|open|>response<|sep|>Use the response tag, then message the handler."
+ "<|close|>response<|sep|><|close|>message<|sep|>")
+ .expect(simple_assist_msg("Use the response tag, then message the handler."))
+ .run();
+
+ // Truncated mid-reasoning (hit the token budget): keep the reasoning.
+ tst.test("I was still thinking when the budget ran out")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .expect_reasoning("I was still thinking when the budget ran out")
+ .run();
+
+ // Single tool call, one argument.
+ tst.test("<|open|>response<|sep|><|close|>response<|sep|>"
+ "<|open|>tools<|sep|>"
+ "<|open|>call tool=\"special_function\" index=\"1\"<|sep|>"
+ "<|open|>argument key=\"arg1\" type=\"number\"<|sep|>1<|close|>argument<|sep|>"
+ "<|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>")
+ .tools({ special_function_tool })
+ .expect_tool_calls({
+ { "special_function", R"({"arg1":1})", "" },
+ })
+ .run();
+
+ // Tool call preceded by reasoning (no opening think tag) and content.
+ tst.test("I should call it<|close|>think<|sep|>"
+ "<|open|>response<|sep|>On it.<|close|>response<|sep|>"
+ "<|open|>tools<|sep|>"
+ "<|open|>call tool=\"special_function\" index=\"1\"<|sep|>"
+ "<|open|>argument key=\"arg1\" type=\"number\"<|sep|>1<|close|>argument<|sep|>"
+ "<|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>")
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .tools({ special_function_tool })
+ .expect(simple_assist_msg("On it.", "I should call it", "special_function",
+ R"({"arg1":1})", ""))
+ .run();
+
+ // Multiple typed arguments: values must come back as JSON numbers, not strings
+ tst.test("<|open|>response<|sep|><|close|>response<|sep|>"
+ "<|open|>tools<|sep|>"
+ "<|open|>call tool=\"special_function_with_opt\" index=\"1\"<|sep|>"
+ "<|open|>argument key=\"arg1\" type=\"number\"<|sep|>1<|close|>argument<|sep|>"
+ "<|open|>argument key=\"arg2\" type=\"number\"<|sep|>2<|close|>argument<|sep|>"
+ "<|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>")
+ .tools({ special_function_tool_with_optional_param })
+ .expect_tool_calls({
+ { "special_function_with_opt", R"({"arg1":1,"arg2":2})", "" },
+ })
+ .run();
+
+ // Parallel tool calls in one <|open|>tools<|sep|> section.
+ tst.test("<|open|>response<|sep|><|close|>response<|sep|>"
+ "<|open|>tools<|sep|>"
+ "<|open|>call tool=\"special_function\" index=\"1\"<|sep|>"
+ "<|open|>argument key=\"arg1\" type=\"number\"<|sep|>1<|close|>argument<|sep|>"
+ "<|close|>call<|sep|>"
+ "<|open|>call tool=\"special_function_with_opt\" index=\"2\"<|sep|>"
+ "<|open|>argument key=\"arg1\" type=\"number\"<|sep|>1<|close|>argument<|sep|>"
+ "<|open|>argument key=\"arg2\" type=\"number\"<|sep|>2<|close|>argument<|sep|>"
+ "<|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>")
+ .parallel_tool_calls(true)
+ .tools({ special_function_tool, special_function_tool_with_optional_param })
+ .expect_tool_calls({
+ { "special_function", R"({"arg1":1})", "" },
+ { "special_function_with_opt", R"({"arg1":1,"arg2":2})", "" },
+ })
+ .run();
+
+ // String-typed argument keeps its literal text (no JSON coercion).
+ tst.test("<|open|>response<|sep|><|close|>response<|sep|>"
+ "<|open|>tools<|sep|>"
+ "<|open|>call tool=\"python\" index=\"1\"<|sep|>"
+ "<|open|>argument key=\"code\" type=\"string\"<|sep|>print('hey')"
+ "<|close|>argument<|sep|>"
+ "<|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>")
+ .tools({ python_tool })
+ .expect_tool_calls({
+ // custom delimiter: the payload itself contains )"
+ { "python", R"JSON({"code":"print('hey')"})JSON", "" },
+ })
+ .run();
+ }
+
// Kimi-K2-Thinking tests - custom parser
// Unique feature: tool call ID embeds function name as functions.<name>:<counter>
{
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index f28be754fd..9105c184ac 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -105,6 +105,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_KIMI_LINEAR
+ || arch == LLM_ARCH_KIMI_K3
|| arch == LLM_ARCH_MISTRAL4) {
n_embd = 128;
n_head = 1;
@@ -145,7 +146,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
- arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR) {
+ arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_KIMI_K3) {
GGML_ASSERT(n_layer >= 2);
std::vector<uint32_t> n_head_per_layer;
n_head_per_layer.reserve(n_layer);
@@ -164,6 +165,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_KIMI_LINEAR
+ || arch == LLM_ARCH_KIMI_K3
|| arch == LLM_ARCH_MISTRAL4) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, uint32_t(576));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(512));
@@ -218,6 +220,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
if (moe) {
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);
ms.add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, n_ff / 2); // distinct from n_ff so a saver key-clobber surfaces on reload
+ ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, n_ff);
ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(1));
@@ -244,6 +247,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
+ ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));
+ ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);
+ ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f);
+ ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
for (uint32_t il = 0; il < n_layer; il++) {
ggml_tensor t;
@@ -372,6 +379,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_PADDLEOCR:
case LLM_ARCH_MIMO2:
case LLM_ARCH_KIMI_LINEAR:
+ case LLM_ARCH_KIMI_K3:
case LLM_ARCH_STEP35:
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_MELLUM: