LLM inference in C/C++
C++ ggml
Model-Saver: Write the SWA pattern, 15 more architectures roundtrip (#29042)
Pascal542348a35c9e
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cppindex 0a27367c98..0f5155b2e4 100644--- a/src/llama-model-saver.cpp+++ b/src/llama-model-saver.cpp@@ -15,26 +15,11 @@bool llama_model_saver_supports_arch(llm_arch arch) {switch (arch) {- case LLM_ARCH_PLAMO3:- case LLM_ARCH_GEMMA3:case LLM_ARCH_GEMMA3N:- case LLM_ARCH_COHERE2:- case LLM_ARCH_COHERE2MOE:- case LLM_ARCH_OLMO2:case LLM_ARCH_BITNET:case LLM_ARCH_T5:- case LLM_ARCH_EXAONE_MOE:- case LLM_ARCH_AFMOE:case LLM_ARCH_APERTUS:- case LLM_ARCH_MIMO2:case LLM_ARCH_STEP35:- case LLM_ARCH_SPARK2_5:- case LLM_ARCH_MUSE_GLIMMER:- case LLM_ARCH_MELLUM:- case LLM_ARCH_LAGUNA:- case LLM_ARCH_GRANITE_SWA:- case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config- case LLM_ARCH_MAPLE:return false;default:return true;@@ -290,7 +275,11 @@ void llama_model_saver::add_kv_from_model() {add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true);add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);- // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???);+ if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {+ // never collapsed to a scalar: the loaders read a scalar as a period+ add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>(+ hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all));+ }add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale);add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale);@@ -300,6 +289,9 @@ void llama_model_saver::add_kv_from_model() {add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa);add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa);+ add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, hparams.n_embd_head_k_mla_swa);+ add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa);+ add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, hparams.n_lora_kv_swa);add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);diff --git a/src/llama-model.cpp b/src/llama-model.cppindex de3b2e38ff..e195f50d0b 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -3305,6 +3305,15 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,}}+void llama_model_base::load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first) {+ if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) {+ return;+ }++ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern, false);+ hparams.set_swa_pattern(n_pattern, dense_first);+}+const int32_t * llama_model_target_layer_ids(const struct llama_model * model) {const auto & v = model->target_layer_ids;return v.empty() ? nullptr : v.data();diff --git a/src/llama-model.h b/src/llama-model.hindex d61afa2cb3..984b2cf388 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -814,6 +814,9 @@ struct llama_model_base : public llama_model {int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,int flags);+ // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern+ void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false);+void load_stats (llama_model_loader & ml) override;void load_hparams(llama_model_loader & ml) override;void load_vocab (llama_model_loader & ml) override;diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cppindex cf02203671..08c22b6ed8 100644--- a/src/models/afmoe.cpp+++ b/src/models/afmoe.cpp@@ -14,9 +14,7 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) {// Pattern: 3 sliding - 1 full (global_attn_every_n_layers = 4)if (hparams.n_swa > 0) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 4;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 4);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cppindex e2b3662560..7ad5244e42 100644--- a/src/models/cohere2.cpp+++ b/src/models/cohere2.cpp@@ -2,9 +2,7 @@void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 4;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 4);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cppindex 5e02cd56e7..7704cbb872 100644--- a/src/models/cohere2moe.cpp+++ b/src/models/cohere2moe.cpp@@ -25,12 +25,7 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {}hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 4;- if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) {- hparams.set_swa_pattern(swa_period, true);- } else {- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());- }+ load_swa_pattern(ml, 4, true);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cppindex 976ee050ad..840da5f678 100644--- a/src/models/exaone-moe.cpp+++ b/src/models/exaone-moe.cpp@@ -3,9 +3,7 @@void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;hparams.n_swa = 128;- uint32_t swa_period = 4;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 4);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cppindex 9ba978956d..c1c55957a4 100644--- a/src/models/exaone4.cpp+++ b/src/models/exaone4.cpp@@ -4,9 +4,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) {if (hparams.n_layer() == 64) { // 32Bhparams.swa_type = LLAMA_SWA_TYPE_STANDARD;hparams.n_swa = 4096;- uint32_t swa_period = 4;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 4);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cppindex 80ed3b1a46..6c97883d82 100644--- a/src/models/gemma-embedding.cpp+++ b/src/models/gemma-embedding.cpp@@ -2,9 +2,7 @@void llama_model_gemma_embedding::load_arch_hparams(llama_model_loader & ml) {hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;- uint32_t swa_period = 6;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 6);hparams.causal_attn = false; // embeddings do not use causal attentiondiff --git a/src/models/gemma2.cpp b/src/models/gemma2.cppindex 2fbfb15a94..9e52434655 100644--- a/src/models/gemma2.cpp+++ b/src/models/gemma2.cpp@@ -3,9 +3,7 @@void llama_model_gemma2::load_arch_hparams(llama_model_loader & ml) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;hparams.n_swa = 4096; // default value of gemma 2- uint32_t swa_period = 2;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 2);hparams.attn_soft_cap = true;hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cppindex 690194529e..f99bbaacd8 100644--- a/src/models/gemma3.cpp+++ b/src/models/gemma3.cpp@@ -4,9 +4,7 @@ void llama_model_gemma3::load_arch_hparams(llama_model_loader & ml) {const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);if (found_swa && hparams.n_swa > 0) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 6;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 6);ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);} else {diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cppindex bb628203aa..4d47ddc62f 100644--- a/src/models/gemma3n.cpp+++ b/src/models/gemma3n.cpp@@ -1,10 +1,8 @@#include "models.h"void llama_model_gemma3n::load_arch_hparams(llama_model_loader & ml) {- uint32_t swa_period = 5;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 5);hparams.n_layer_kv_from_start = 20;hparams.f_attention_scale = 1.0f;diff --git a/src/models/laguna.cpp b/src/models/laguna.cppindex 556400bfce..2bc4fa8a0f 100644--- a/src/models/laguna.cpp+++ b/src/models/laguna.cpp@@ -36,9 +36,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) {if (hparams.n_swa > 0) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 4;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period, /*dense_first=*/true); // XS.2: FULL at il%4==0+ load_swa_pattern(ml, 4, /*dense_first=*/true); // XS.2: FULL at il%4==0// Per-layer-type RoPE: full layers use YaRN θ=500000 over 64 dims;// SWA layers use default RoPE θ=10000 over 128 dims. Base load_hparamsdiff --git a/src/models/llama4.cpp b/src/models/llama4.cppindex 8a812beffa..4f79b4d1ee 100644--- a/src/models/llama4.cpp+++ b/src/models/llama4.cpp@@ -16,9 +16,7 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) {hparams.f_attn_temp_scale = 0.1f;hparams.f_attn_temp_offset = 1.0f;- uint32_t swa_period = 4; // pattern: 3 chunked - 1 full- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 4); // pattern: 3 chunked - 1 fullhparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/mellum.cpp b/src/models/mellum.cppindex 872a9c8f55..3ab79747af 100644--- a/src/models/mellum.cpp+++ b/src/models/mellum.cpp@@ -8,13 +8,7 @@ void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) {if (hparams.n_swa > 0) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 4;- const auto res = ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- if (res) {- hparams.set_swa_pattern(swa_period);- } else {- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());- }+ load_swa_pattern(ml, 4);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cppindex f3e9407e01..b7542d59bd 100644--- a/src/models/modern-bert.cpp+++ b/src/models/modern-bert.cpp@@ -5,9 +5,7 @@ void llama_model_modern_bert::load_arch_hparams(llama_model_loader & ml) {if (found_swa && hparams.n_swa > 0) {hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);- uint32_t swa_period = 3;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period, true);+ load_swa_pattern(ml, 3, true);} else {hparams.swa_type = LLAMA_SWA_TYPE_NONE;}diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cppindex 0e94153088..0e5f75ebed 100644--- a/src/models/muse-glimmer.cpp+++ b/src/models/muse-glimmer.cpp@@ -10,12 +10,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 4;- if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) {- hparams.set_swa_pattern(swa_period);- } else {- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());- }+ load_swa_pattern(ml, 4);switch (hparams.n_layer()) {case 52: type = LLM_TYPE_30B; break;diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cppindex 05b9394b8f..fe5dc88d44 100644--- a/src/models/olmo2.cpp+++ b/src/models/olmo2.cpp@@ -6,9 +6,7 @@ void llama_model_olmo2::load_arch_hparams(llama_model_loader & ml) {const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);if (found_swa && hparams.n_swa > 0) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 4;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 4);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = 1.0; // See olmo2.cppdiff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cppindex c9f9b677d0..39660a325d 100644--- a/src/models/openai-moe.cpp+++ b/src/models/openai-moe.cpp@@ -6,9 +6,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;- uint32_t swa_period = 2;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 2);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cppindex 16d0b1dcef..f8235f8a83 100644--- a/src/models/plamo3.cpp+++ b/src/models/plamo3.cpp@@ -6,9 +6,7 @@ void llama_model_plamo3::load_arch_hparams(llama_model_loader & ml) {if (found_swa && hparams.n_swa > 0) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);- uint32_t swa_period = 8;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period);+ load_swa_pattern(ml, 8);} else {hparams.swa_type = LLAMA_SWA_TYPE_NONE;}diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cppindex 680ffb8fda..555f8b7184 100644--- a/src/models/smallthinker.cpp+++ b/src/models/smallthinker.cpp@@ -6,9 +6,7 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) {if (found_swa && hparams.n_swa > 0) {hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;hparams.n_swa = 4096;- uint32_t swa_period = 4;- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);- hparams.set_swa_pattern(swa_period, true);+ load_swa_pattern(ml, 4, true);hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;