Skip to content

LLM inference in C/C++

C++ ggml

Model-Saver: Write the SWA pattern, 15 more architectures roundtrip (#29042)

Pascal542348a35c9e
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 0a27367c98..0f5155b2e4 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -15,26 +15,11 @@
bool llama_model_saver_supports_arch(llm_arch arch) {
switch (arch) {
- case LLM_ARCH_PLAMO3:
- case LLM_ARCH_GEMMA3:
case LLM_ARCH_GEMMA3N:
- case LLM_ARCH_COHERE2:
- case LLM_ARCH_COHERE2MOE:
- case LLM_ARCH_OLMO2:
case LLM_ARCH_BITNET:
case LLM_ARCH_T5:
- case LLM_ARCH_EXAONE_MOE:
- case LLM_ARCH_AFMOE:
case LLM_ARCH_APERTUS:
- case LLM_ARCH_MIMO2:
case LLM_ARCH_STEP35:
- case LLM_ARCH_SPARK2_5:
- case LLM_ARCH_MUSE_GLIMMER:
- case LLM_ARCH_MELLUM:
- case LLM_ARCH_LAGUNA:
- case LLM_ARCH_GRANITE_SWA:
- case LLM_ARCH_DOTS3NOTE: // TODO: need to handle SWA pattern and MLA+SWA config
- case LLM_ARCH_MAPLE:
return false;
default:
return true;
@@ -290,7 +275,11 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);
add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true);
add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
- // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???);
+ if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
+ // never collapsed to a scalar: the loaders read a scalar as a period
+ add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>(
+ hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all));
+ }
add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);
add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale);
add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale);
@@ -300,6 +289,9 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);
add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa);
add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa);
+ add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, hparams.n_embd_head_k_mla_swa);
+ add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa);
+ add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, hparams.n_lora_kv_swa);
add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index de3b2e38ff..e195f50d0b 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -3305,6 +3305,15 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,
}
}
+void llama_model_base::load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first) {
+ if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) {
+ return;
+ }
+
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern, false);
+ hparams.set_swa_pattern(n_pattern, dense_first);
+}
+
const int32_t * llama_model_target_layer_ids(const struct llama_model * model) {
const auto & v = model->target_layer_ids;
return v.empty() ? nullptr : v.data();
diff --git a/src/llama-model.h b/src/llama-model.h
index d61afa2cb3..984b2cf388 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -814,6 +814,9 @@ struct llama_model_base : public llama_model {
int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
int flags);
+ // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern
+ void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false);
+
void load_stats (llama_model_loader & ml) override;
void load_hparams(llama_model_loader & ml) override;
void load_vocab (llama_model_loader & ml) override;
diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp
index cf02203671..08c22b6ed8 100644
--- a/src/models/afmoe.cpp
+++ b/src/models/afmoe.cpp
@@ -14,9 +14,7 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) {
// Pattern: 3 sliding - 1 full (global_attn_every_n_layers = 4)
if (hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 4;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 4);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp
index e2b3662560..7ad5244e42 100644
--- a/src/models/cohere2.cpp
+++ b/src/models/cohere2.cpp
@@ -2,9 +2,7 @@
void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 4;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 4);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp
index 5e02cd56e7..7704cbb872 100644
--- a/src/models/cohere2moe.cpp
+++ b/src/models/cohere2moe.cpp
@@ -25,12 +25,7 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {
}
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 4;
- if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) {
- hparams.set_swa_pattern(swa_period, true);
- } else {
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
- }
+ load_swa_pattern(ml, 4, true);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp
index 976ee050ad..840da5f678 100644
--- a/src/models/exaone-moe.cpp
+++ b/src/models/exaone-moe.cpp
@@ -3,9 +3,7 @@
void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.n_swa = 128;
- uint32_t swa_period = 4;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 4);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp
index 9ba978956d..c1c55957a4 100644
--- a/src/models/exaone4.cpp
+++ b/src/models/exaone4.cpp
@@ -4,9 +4,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) {
if (hparams.n_layer() == 64) { // 32B
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.n_swa = 4096;
- uint32_t swa_period = 4;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 4);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp
index 80ed3b1a46..6c97883d82 100644
--- a/src/models/gemma-embedding.cpp
+++ b/src/models/gemma-embedding.cpp
@@ -2,9 +2,7 @@
void llama_model_gemma_embedding::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;
- uint32_t swa_period = 6;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 6);
hparams.causal_attn = false; // embeddings do not use causal attention
diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp
index 2fbfb15a94..9e52434655 100644
--- a/src/models/gemma2.cpp
+++ b/src/models/gemma2.cpp
@@ -3,9 +3,7 @@
void llama_model_gemma2::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.n_swa = 4096; // default value of gemma 2
- uint32_t swa_period = 2;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 2);
hparams.attn_soft_cap = true;
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp
index 690194529e..f99bbaacd8 100644
--- a/src/models/gemma3.cpp
+++ b/src/models/gemma3.cpp
@@ -4,9 +4,7 @@ void llama_model_gemma3::load_arch_hparams(llama_model_loader & ml) {
const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);
if (found_swa && hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 6;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 6);
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
} else {
diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp
index bb628203aa..4d47ddc62f 100644
--- a/src/models/gemma3n.cpp
+++ b/src/models/gemma3n.cpp
@@ -1,10 +1,8 @@
#include "models.h"
void llama_model_gemma3n::load_arch_hparams(llama_model_loader & ml) {
- uint32_t swa_period = 5;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 5);
hparams.n_layer_kv_from_start = 20;
hparams.f_attention_scale = 1.0f;
diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp
index 556400bfce..2bc4fa8a0f 100644
--- a/src/models/laguna.cpp
+++ b/src/models/laguna.cpp
@@ -36,9 +36,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) {
if (hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 4;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period, /*dense_first=*/true); // XS.2: FULL at il%4==0
+ load_swa_pattern(ml, 4, /*dense_first=*/true); // XS.2: FULL at il%4==0
// Per-layer-type RoPE: full layers use YaRN θ=500000 over 64 dims;
// SWA layers use default RoPE θ=10000 over 128 dims. Base load_hparams
diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp
index 8a812beffa..4f79b4d1ee 100644
--- a/src/models/llama4.cpp
+++ b/src/models/llama4.cpp
@@ -16,9 +16,7 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) {
hparams.f_attn_temp_scale = 0.1f;
hparams.f_attn_temp_offset = 1.0f;
- uint32_t swa_period = 4; // pattern: 3 chunked - 1 full
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 4); // pattern: 3 chunked - 1 full
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp
index 872a9c8f55..3ab79747af 100644
--- a/src/models/mellum.cpp
+++ b/src/models/mellum.cpp
@@ -8,13 +8,7 @@ void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) {
if (hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 4;
- const auto res = ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- if (res) {
- hparams.set_swa_pattern(swa_period);
- } else {
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
- }
+ load_swa_pattern(ml, 4);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp
index f3e9407e01..b7542d59bd 100644
--- a/src/models/modern-bert.cpp
+++ b/src/models/modern-bert.cpp
@@ -5,9 +5,7 @@ void llama_model_modern_bert::load_arch_hparams(llama_model_loader & ml) {
if (found_swa && hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
- uint32_t swa_period = 3;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period, true);
+ load_swa_pattern(ml, 3, true);
} else {
hparams.swa_type = LLAMA_SWA_TYPE_NONE;
}
diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cpp
index 0e94153088..0e5f75ebed 100644
--- a/src/models/muse-glimmer.cpp
+++ b/src/models/muse-glimmer.cpp
@@ -10,12 +10,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 4;
- if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) {
- hparams.set_swa_pattern(swa_period);
- } else {
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
- }
+ load_swa_pattern(ml, 4);
switch (hparams.n_layer()) {
case 52: type = LLM_TYPE_30B; break;
diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp
index 05b9394b8f..fe5dc88d44 100644
--- a/src/models/olmo2.cpp
+++ b/src/models/olmo2.cpp
@@ -6,9 +6,7 @@ void llama_model_olmo2::load_arch_hparams(llama_model_loader & ml) {
const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);
if (found_swa && hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 4;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 4);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = 1.0; // See olmo2.cpp
diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp
index c9f9b677d0..39660a325d 100644
--- a/src/models/openai-moe.cpp
+++ b/src/models/openai-moe.cpp
@@ -6,9 +6,7 @@ void llama_model_openai_moe::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
- uint32_t swa_period = 2;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 2);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp
index 16d0b1dcef..f8235f8a83 100644
--- a/src/models/plamo3.cpp
+++ b/src/models/plamo3.cpp
@@ -6,9 +6,7 @@ void llama_model_plamo3::load_arch_hparams(llama_model_loader & ml) {
if (found_swa && hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
- uint32_t swa_period = 8;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period);
+ load_swa_pattern(ml, 8);
} else {
hparams.swa_type = LLAMA_SWA_TYPE_NONE;
}
diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp
index 680ffb8fda..555f8b7184 100644
--- a/src/models/smallthinker.cpp
+++ b/src/models/smallthinker.cpp
@@ -6,9 +6,7 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) {
if (found_swa && hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.n_swa = 4096;
- uint32_t swa_period = 4;
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
- hparams.set_swa_pattern(swa_period, true);
+ load_swa_pattern(ml, 4, true);
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;