LLM inference in C/C++
C++ ggml
models : add Flint model (FlintForCausalLM) (#30928)
alexey-rsia37cc46104eb
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 369ce7afda..8921f53e44 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -20,6 +20,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"NEOChatModel": "sensenova_u1","AfmoeForCausalLM": "afmoe","LagunaForCausalLM": "laguna",+ "FlintForCausalLM": "flint","ApertusForCausalLM": "llama","ArceeForCausalLM": "llama","ArcticForCausalLM": "arctic",diff --git a/conversion/base.py b/conversion/base.pyindex 2cbaed700f..f3bbe7b0ff 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -1701,7 +1701,7 @@ class TextModel(ModelBase):self.gguf_writer.add_expert_group_used_count(n_group_used)logger.info(f"gguf: expert groups used count = {n_group_used}")- if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func"], optional=True)) is not None:+ if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func", "router_score_function"], optional=True)) is not None:if score_func == "sigmoid":self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)elif score_func == "softmax":@@ -1939,6 +1939,9 @@ class TextModel(ModelBase):if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e":# ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0res = "cohere2moe"+ if chkhsh == "d5420e601e979078c670bc503d9ac411c311fa76b76a76267c31e0d6ba816637":+ # ref: https://huggingface.co/RecursiveSI/flint+ res = "flint"if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":# ref: https://huggingface.co/Qwen/Qwen1.5-7Bres = "qwen2"diff --git a/conversion/flint.py b/conversion/flint.pynew file mode 100644index 0000000000..a69314a9bc--- /dev/null+++ b/conversion/flint.py@@ -0,0 +1,38 @@+from __future__ import annotations++from .base import ModelBase, gguf+from .qwen import Qwen2MoeModel++[email protected]("FlintForCausalLM")[email protected]("RecursiveSI/flint")+class FlintModel(Qwen2MoeModel):+ model_arch = gguf.MODEL_ARCH.FLINT++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ dense = self.hparams["mlp_only_layers"]+ if dense != list(range(len(dense))):+ raise ValueError("Flint dense layers must precede the MoE layers")+ self.gguf_writer.add_leading_dense_block_count(len(dense))+ self.gguf_writer.add_expert_shared_count(+ self.hparams["shared_expert_intermediate_size"] // self.hparams["moe_intermediate_size"]+ )+ self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])+ self.gguf_writer.add_expert_weights_scale(1.0)+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])+ self.gguf_writer.add_sliding_window_pattern([+ kind == "sliding_attention" for kind in self.hparams["layer_types"]+ ])+ self.gguf_writer.add_rope_pattern([+ i not in self.hparams["no_rope_layer_indices"] for i in range(self.block_count)+ ])++ def modify_tensors(self, data_torch, name, bid):+ if name.endswith(".softmax_offset"):+ name += ".weight"+ if name.endswith(".gate.expert_bias"):+ if not self.hparams["router_expert_bias"]:+ return+ name += ".bias"+ yield from super().modify_tensors(data_torch, name, bid)diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.pyindex d3aa3c3807..7d8c4fda64 100755--- a/convert_hf_to_gguf_update.py+++ b/convert_hf_to_gguf_update.py@@ -101,6 +101,7 @@ models = [{"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },{"name": "tiny_aya", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", },{"name": "cohere2moe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0", },+ {"name": "flint", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/RecursiveSI/flint", },{"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },{"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },{"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 619a15fc96..51a16ef52b 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -577,6 +577,7 @@ class MODEL_ARCH(IntEnum):SENSENOVA_U1 = auto()QWEN3 = auto()QWEN3MOE = auto()+ FLINT = auto()QWEN3NEXT = auto()HRM_TEXT = auto()BEAM = auto()@@ -1394,6 +1395,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {MODEL_ARCH.QWEN2VL: "qwen2vl",MODEL_ARCH.SENSENOVA_U1: "sensenova_u1",MODEL_ARCH.QWEN3: "qwen3",+ MODEL_ARCH.FLINT: "flint",MODEL_ARCH.QWEN3MOE: "qwen3moe",MODEL_ARCH.QWEN3NEXT: "qwen3next",MODEL_ARCH.HRM_TEXT: "hrm_text",@@ -2974,6 +2976,33 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.FFN_DOWN_EXP,MODEL_TENSOR.FFN_UP_EXP,],++ MODEL_ARCH.FLINT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ ],MODEL_ARCH.QWEN3NEXT: [MODEL_TENSOR.TOKEN_EMBD,MODEL_TENSOR.OUTPUT_NORM,diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex 3586d65285..79c9add78a 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -429,6 +429,7 @@ class TensorNameMap:),MODEL_TENSOR.ATTN_SINKS: (+ "model.layers.{bid}.self_attn.softmax_offset", # flint"model.layers.{bid}.self_attn.sinks", # openai-moe"model.layers.{bid}.self_attn.attention_sink_bias", # mimov2"model.layers.{bid}.self_attn.learnable_sink_param", # hy-v4diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 1c11536b51..62b012c5c3 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -35,6 +35,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_QWEN2VL, "qwen2vl" },{ LLM_ARCH_SENSENOVA_U1, "sensenova_u1" },{ LLM_ARCH_QWEN3, "qwen3" },+ { LLM_ARCH_FLINT, "flint" },{ LLM_ARCH_QWEN3MOE, "qwen3moe" },{ LLM_ARCH_QWEN3NEXT, "qwen3next" },{ LLM_ARCH_QWEN3VL, "qwen3vl" },diff --git a/src/llama-arch.h b/src/llama-arch.hindex afceb8ea6f..6713db6b59 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -41,6 +41,7 @@ enum llm_arch {LLM_ARCH_SENSENOVA_U1,LLM_ARCH_QWEN3,LLM_ARCH_QWEN3MOE,+ LLM_ARCH_FLINT,LLM_ARCH_QWEN3NEXT,LLM_ARCH_QWEN3VL,LLM_ARCH_QWEN3VLMOE,diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex 8c31b51307..f19368693f 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -1684,7 +1684,10 @@ llm_graph_amax_src1 llm_graph_context::build_amax_src1_f16(constexpr float k_ln_2 = 0.6931471805599453f;ggml_tensor * e;- e = ggml_scale(ctx0, amax, 1.0f/32768.0f);+ // note: this can technically be:+ // e = ggml_scale(ctx0, amax, 1.0f/32768.0f);+ // but we leave x32 headroom for the F16 sums of 32 values in Q8 activation blocks+ e = ggml_scale(ctx0, amax, 1.0f/1024.0f);e = ggml_log (ctx0, e);e = ggml_scale(ctx0, e, k_log2_e);e = ggml_ceil (ctx0, e);@@ -2505,7 +2508,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(// ref: https://github.com/ggml-org/llama.cpp/pull/30656#issuecomment-6269597922llm_graph_amax_src1 amax_src1 = { nullptr, nullptr };- if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM) {+ if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT) {amax_src1 = build_amax_src1_f16(cur);cur = amax_src1.res;}diff --git a/src/llama-graph.h b/src/llama-graph.hindex 9c5f3aea9a..a49f5432d2 100644--- a/src/llama-graph.h+++ b/src/llama-graph.h@@ -1122,7 +1122,7 @@ struct llm_graph_context {ggml_tensor * slots = nullptr) const;// rescale `cur` (the src1 of a matmul) by a power of two, computed from its maximum absolute- // when the values already fit in F16, the factors are exactly 1.0f+ // keep both individual values and Q8 activation block sums within F16 rangellm_graph_amax_src1 build_amax_src1_f16(ggml_tensor * cur) const;diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cppindex b7dfadef46..6f0e3f7f79 100644--- a/src/llama-kv-cache.cpp+++ b/src/llama-kv-cache.cpp@@ -882,7 +882,10 @@ bool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_coLLAMA_LOG_DEBUG("%s: applying K-shift\n", __func__);// apply K-shift if needed- if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE) {+ if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE &&+ std::any_of(layers.begin(), layers.end(), [&](const auto & layer) {+ return hparams.has_rope(layer.il);+ })) {ggml_backend_sched_reset(sched);auto * res = lctx->get_gf_res_reserve();diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 3797c567dc..60199d6d99 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -114,6 +114,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_paramsreturn new llama_model_sensenova_u1(params);case LLM_ARCH_QWEN3:return new llama_model_qwen3(params);+ case LLM_ARCH_FLINT:+ return new llama_model_flint(params);case LLM_ARCH_QWEN3MOE:return new llama_model_qwen3moe(params);case LLM_ARCH_QWEN3VL:@@ -3571,6 +3573,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {case LLM_ARCH_QWEN2MOE:case LLM_ARCH_QWEN3:case LLM_ARCH_VLLM_DECISION2:+ case LLM_ARCH_FLINT:case LLM_ARCH_QWEN3MOE:case LLM_ARCH_LLADA_MOE:case LLM_ARCH_RND1:diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex a3d5dedef3..05c0f96a29 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -281,6 +281,13 @@ struct llm_tokenizer_bpe : llm_tokenizer {llm_tokenizer_bpe(const llama_vocab & vocab) {GGML_ASSERT(vocab.get_type() == LLAMA_VOCAB_TYPE_BPE);switch (vocab.get_pre_type()) {+ case LLAMA_VOCAB_PRE_TYPE_FLINT:+ regex_exprs = {+ "\\p{N}{1,3}",+ "[一-龥-ゟ゠-ヿ]+",+ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?| ?[\\p{P}\\p{S}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",+ };+ break;case LLAMA_VOCAB_PRE_TYPE_LLAMA3:regex_exprs = {// original regex from tokenizer.json@@ -2417,6 +2424,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {LLAMA_LOG_WARN("%s: ************************************ \n", __func__);LLAMA_LOG_WARN("%s: \n", __func__);pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;+ } else if (tokenizer_pre == "flint") {+ pre_type = LLAMA_VOCAB_PRE_TYPE_FLINT;+ clean_spaces = false;} else if (tokenizer_pre == "default") {pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;} else if (tokenizer_pre == "minicpm5") {@@ -3147,7 +3157,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {// this is currently determined based on the token text, which is obviously not ideal// ref: https://github.com/ggml-org/llama.cpp/issues/9606special_eog_ids.clear();-if (special_fim_pad_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_pad_id) == 0) {special_eog_ids.insert(special_fim_pad_id);}@@ -3181,6 +3190,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {|| t.first == "[EOT]" // Kimi-K2|| t.first == "[EOS]" // Kimi-K2|| t.first == "<|end_of_text|>"+ || (pre_type == LLAMA_VOCAB_PRE_TYPE_FLINT && (t.first == "<|user|>" || t.first == "<|observation|>"))|| t.first == "<end_of_utterance>" // smoldocling|| t.first == "<eos>" // gemma4|| t.first == "<turn|>" // gemma4diff --git a/src/llama-vocab.h b/src/llama-vocab.hindex 415acd22bf..a87c0de42b 100644--- a/src/llama-vocab.h+++ b/src/llama-vocab.h@@ -70,6 +70,7 @@ enum llama_vocab_pre_type {LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59,LLAMA_VOCAB_PRE_TYPE_MMBERT = 60,LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61,+ LLAMA_VOCAB_PRE_TYPE_FLINT = 62,};struct LLM_KV;diff --git a/src/models/flint.cpp b/src/models/flint.cppnew file mode 100644index 0000000000..e7e6f6b792--- /dev/null+++ b/src/models/flint.cpp@@ -0,0 +1,194 @@+#include "models.h"++void llama_model_flint::load_arch_hparams(llama_model_loader & ml) {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);+ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);+ ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);++ ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);+ ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);+ ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern);+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;+ type = LLM_TYPE_UNKNOWN;+}++void llama_model_flint::load_arch_tensors(llama_model_loader &) {+ LLAMA_LOAD_LOCALS;++ const int64_t n_ff_exp = hparams.n_ff_exp();++ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);++ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);++ for (int i = 0; i < n_layer; ++i) {+ auto & layer = layers[i];++ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);++ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);++ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);++ layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0);++ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);++ if (i < (int) hparams.n_layer_dense_lead) {+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);+ } else {+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);++ if (n_expert == 0) {+ throw std::runtime_error("n_expert must be > 0");+ }+ if (n_expert_used == 0) {+ throw std::runtime_error("n_expert_used must be > 0");+ }++ // MoE branch+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);++ // Shared expert branch+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0);+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { hparams.n_ff_shexp, n_embd}, 0);+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0);+ }+ }+}++std::unique_ptr<llm_graph_context> llama_model_flint::build_arch_graph(const llm_graph_params & params) const {+ return std::make_unique<graph>(*this, params);+}++llama_model_flint::graph::graph(const llama_model & model, const llm_graph_params & params) :+ llm_graph_context(params) {+ const int64_t n_embd_head = hparams.n_embd_head_v();++ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());+ GGML_ASSERT(n_embd_head == n_rot);++ ggml_tensor * cur;+ ggml_tensor * inpL;++ inpL = build_inp_embd(model.tok_embd);++ // inp_pos - contains the positions+ ggml_tensor * inp_pos = build_inp_pos();++ auto * inp_attn = build_attn_inp_kv_iswa();++ ggml_tensor * inp_out_ids = build_inp_out_ids();++ for (int il = 0; il < n_layer; ++il) {+ res->t_layer_inp[il] = inpL;+ ggml_tensor * inpSA = inpL;++ // norm+ cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);+ cb(cur, "attn_norm", il);++ // self_attention+ {+ // compute Q and K and RoPE them+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,+ n_embd_head, n_head, n_head_kv, il);++ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);+ Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);+ if (hparams.has_rope(il)) {+ Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ }+ cb(Qcur, "Qcur", il);+ cb(Kcur, "Kcur", il);+ cb(Vcur, "Vcur", il);++ cur = build_attn(inp_attn,+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,+ Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);+ }+ if (il == n_layer - 1 && inp_out_ids) {+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+ }+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // MoE branch+ cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);+ cb(cur, "ffn_norm", il);++ if ((uint32_t) il < hparams.n_layer_dense_lead) {+ cur = build_ffn(cur,+ model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,+ model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,+ model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,+ NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);+ cb(cur, "ffn_out", il);+ } else {+ ggml_tensor * moe_out = build_moe_ffn(cur,+ model.layers[il].ffn_gate_inp,+ model.layers[il].ffn_up_exps,+ model.layers[il].ffn_gate_exps,+ model.layers[il].ffn_down_exps,+ model.layers[il].ffn_exp_probs_b,+ n_expert, n_expert_used,+ LLM_FFN_SILU, hparams.expert_weights_norm,+ hparams.expert_weights_scale,+ (llama_expert_gating_func_type) hparams.expert_gating_func,+ il, nullptr, nullptr,+ model.layers[il].ffn_up_exps_s,+ model.layers[il].ffn_gate_exps_s,+ model.layers[il].ffn_down_exps_s);+ cb(moe_out, "ffn_moe_out", il);++ {+ ggml_tensor * ffn_shexp =+ build_ffn(cur,+ model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,+ model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,+ model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,+ NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);+ cb(ffn_shexp, "ffn_shexp", il);++ cur = ggml_add(ctx0, moe_out, ffn_shexp);+ cb(cur, "ffn_out", il);+ }+ }+ cur = ggml_add(ctx0, cur, ffn_inp);++ cur = build_cvec(cur, il);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }+ cur = inpL;++ cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);++ cb(cur, "result_norm", -1);+ res->t_embd = cur;++ // lm_head+ cur = build_lora_mm(model.output, cur, model.output_s);++ cb(cur, "result_output", -1);+ res->t_logits = cur;+}diff --git a/src/models/models.h b/src/models/models.hindex 47b0203da1..bdcc1a4fae 100644--- a/src/models/models.h+++ b/src/models/models.h@@ -693,6 +693,19 @@ struct llama_model_qwen3moe : public llama_model_base {};+struct llama_model_flint : public llama_model_base {+ llama_model_flint(const struct llama_model_params & params) : llama_model_base(params) {}+ void load_arch_hparams(llama_model_loader & ml) override;+ void load_arch_tensors(llama_model_loader & ml) override;++ struct graph : public llm_graph_context {+ graph(const llama_model & model, const llm_graph_params & params);+ };++ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;+};++struct llama_model_qwen3vl : public llama_model_base {llama_model_qwen3vl(const struct llama_model_params & params) : llama_model_base(params) {}void load_arch_hparams(llama_model_loader & ml) override;diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csvindex e6cd43a82b..56044b350e 100644--- a/tests/fusion/MTL.csv+++ b/tests/fusion/MTL.csv@@ -71,6 +71,9 @@ falcon ,0 ,any ,ADD+ADD , 2falcon ,0 ,any ,NORM+MUL+ADD , 5falcon-h1 ,0 ,any ,ADD+ADD , 2falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9+flint ,0 ,any ,ADD+ADD , 1+flint ,0 ,any ,MUL+ADD , 2+flint ,0 ,any ,RMS_NORM+MUL , 9gemma ,0 ,any ,RMS_NORM+MUL , 5gemma2 ,0 ,any ,RMS_NORM+MUL , 5gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex c86b1315a6..1e0b755112 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -361,13 +361,16 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));} else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||- arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE) {+ arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) {std::vector<uint32_t> pattern;pattern.reserve(n_layer);for (uint32_t il = 0; il < n_layer; il++) {pattern.push_back(il % 2);}ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);+ if (arch == LLM_ARCH_FLINT) {+ ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, pattern);+ }} else {ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));}@@ -525,6 +528,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, gating_func); // sigmoid : softmax : sqrtsoftplusms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));+ if (arch == LLM_ARCH_FLINT) {+ ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);+ }if (arch == LLM_ARCH_K2_HORIZON) {ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, uint32_t(2));ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, uint32_t(2));@@ -754,6 +760,7 @@ static bool check_causal_attn_toggle(static bool moe_mandatory(const llm_arch arch) {switch (arch) {+ case LLM_ARCH_FLINT:case LLM_ARCH_LLAMA4:case LLM_ARCH_MAION_CODER:case LLM_ARCH_COHERE2MOE: