LLM inference in C/C++
C++ ggml
model : GraniteSWAForCausalLM / GraniteMoeSWAForCausalLM (#25505)
Gabe Goodhart7221e24f579e
diff --git a/conversion/__init__.py b/conversion/__init__.pyindex 3232a1050b..5ae6ad819f 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -109,6 +109,8 @@ TEXT_MODEL_MAP: dict[str, str] = {"GraniteSwitchForCausalLM": "granite","GraniteSpeechForConditionalGeneration": "granite","GraniteSpeechPlusForConditionalGeneration": "granite",+ "GraniteSWAForCausalLM": "granite",+ "GraniteMoeSWAForCausalLM": "granite","Grok1ForCausalLM": "grok","GrokForCausalLM": "grok","GroveMoeForCausalLM": "grovemoe",diff --git a/conversion/granite.py b/conversion/granite.pyindex 5f1e3e8472..796d37cca2 100644--- a/conversion/granite.py+++ b/conversion/granite.py@@ -74,6 +74,108 @@ class GraniteModel(LlamaModel):return super().filter_tensors(item)[email protected]("GraniteSWAForCausalLM")+class GraniteSWAModel(GraniteModel):+ """Conversion for IBM's GraniteSWAForCausalLM (interleaved sliding window attention)"""+ model_arch = gguf.MODEL_ARCH.GRANITE_SWA++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ name, gen = item++ if name.endswith("sinks"):+ name += ".weight"++ return super().filter_tensors((name, gen))++ def set_gguf_parameters(self):+ """GraniteSWA uses Granite parameters plus sliding window configuration."""+ super().set_gguf_parameters()++ # Add sliding_window from config+ sliding_window = self.hparams.get("sliding_window", 128)+ self.gguf_writer.add_sliding_window(sliding_window)+ logger.info("gguf: (granite_swa) sliding_window = %s", sliding_window)++ # Derive sliding_window_pattern from layer_types+ if layer_types := self.hparams.get("layer_types"):+ is_swa = [t == "sliding_attention" for t in layer_types]+ self.gguf_writer.add_sliding_window_pattern(is_swa)+ logger.info("gguf: (granite_swa) sliding_window_pattern = %d SWA layers / %d total",+ sum(is_swa), len(is_swa))+ else:+ # Fall back to period-based pattern: i % 4 != 0+ # This matches the transformers default pattern+ n_layers = self.block_count+ is_swa = [i % 4 != 0 for i in range(n_layers)]+ self.gguf_writer.add_sliding_window_pattern(is_swa)+ logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total",+ sum(is_swa), n_layers)++ # Add rope_pattern from no_rope_layers+ if no_rope_layers := self.hparams.get("no_rope_layers"):+ # Convert 1/0 to bool (1 = use RoPE, 0 = NoPE)+ rope_pattern = [bool(x) for x in no_rope_layers]+ self.gguf_writer.add_rope_pattern(rope_pattern)+ logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total",+ sum(rope_pattern), len(rope_pattern))++[email protected]("GraniteMoeSWAForCausalLM")+class GraniteMoeSWAModel(GraniteSWAModel):+ """Conversion for IBM's GraniteMoeSWAForCausalLM (unified dense + MoE with iSWA)"""+ model_arch = gguf.MODEL_ARCH.GRANITE_SWA++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ if shared_intermediate_size := self.hparams.get("shared_intermediate_size"):+ self.gguf_writer.add_expert_shared_feed_forward_length(shared_intermediate_size)+ logger.info("gguf: (granitemoewa) shared_intermediate_size = %s", shared_intermediate_size)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ """Split merged MoE tensors (gate+up) following standard MoE pattern."""++ # Handle expert FFN tensors (merged gate+up) - swash format: experts.gate_up_proj+ # Kept fused since inference (build_moe_ffn) supports a single gate_up_exps+ # tensor for the routed experts.+ if name.endswith("block_sparse_moe.experts.gate_up_proj"):+ ffn_dim = self.hparams["intermediate_size"]+ assert data_torch.shape[-2] == 2 * ffn_dim, f"Merged FFN tensor size must be 2 * intermediate_size, got {data_torch.shape[-2]}"+ yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid)+ return++ # Handle expert FFN down projection - swash format: experts.down_proj+ if name.endswith("block_sparse_moe.experts.down_proj"):+ yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid), bid)+ return++ # Handle expert FFN tensors (merged gate+up) - standard granite format: input_linear.weight+ # Kept fused since inference (build_moe_ffn) supports a single gate_up_exps+ # tensor for the routed experts.+ if name.endswith("block_sparse_moe.input_linear.weight"):+ ffn_dim = self.hparams["intermediate_size"]+ assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size"+ yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid)+ return++ # Handle shared expert FFN tensors (if present) - kept fused since+ # inference (build_ffn) supports a single ffn_up_shexp tensor with+ # LLM_FFN_SWIGLU for the shared expert.+ if name.endswith("shared_mlp.input_linear.weight"):+ ffn_dim = self.hparams.get("shared_intermediate_size", self.hparams["intermediate_size"])+ assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size"+ yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), bid)+ return++ # Handle shared expert output (if present)+ if name.endswith("shared_mlp.output_linear.weight"):+ yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, bid), bid)+ return++ # Pass through to parent for all other tensors (including sinks)+ yield from super().modify_tensors(data_torch, name, bid)++@ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM")@ModelBase.example("ibm-granite/granite-3.1-3b-a800m-instruct")class GraniteMoeModel(GraniteModel):diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex d043c9b6ec..fad8d1fd8c 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -208,6 +208,7 @@ class Keys:SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers"SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern"TEMPERATURE_SCALE = "{arch}.attention.temperature_scale"+ ROPE_PATTERN = "{arch}.attention.rope_pattern"class Indexer:HEAD_COUNT = "{arch}.attention.indexer.head_count"@@ -549,6 +550,7 @@ class MODEL_ARCH(IntEnum):GRANITE_MOE = auto()GRANITE_HYBRID = auto()GRANITE_SWITCH = auto()+ GRANITE_SWA = auto()CHAMELEON = auto()WAVTOKENIZER_DEC = auto()PLM = auto()@@ -1265,6 +1267,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {MODEL_ARCH.GRANITE_MOE: "granitemoe",MODEL_ARCH.GRANITE_HYBRID: "granitehybrid",MODEL_ARCH.GRANITE_SWITCH: "graniteswitch",+ MODEL_ARCH.GRANITE_SWA: "granite_swa",MODEL_ARCH.CHAMELEON: "chameleon",MODEL_ARCH.WAVTOKENIZER_DEC: "wavtokenizer-dec",MODEL_ARCH.PLM: "plm",@@ -4152,6 +4155,31 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.FFN_DOWN,MODEL_TENSOR.FFN_UP,],+ MODEL_ARCH.GRANITE_SWA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ # MoE (GraniteMoeSWA)+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ # Shared expert - gate+up kept fused in FFN_UP_SHEXP (LLM_FFN_SWIGLU)+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ ],MODEL_ARCH.CHAMELEON: [MODEL_TENSOR.TOKEN_EMBD,MODEL_TENSOR.OUTPUT_NORM,diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex 9e0914fd86..16ae9f999d 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -824,6 +824,9 @@ class GGUFWriter:else:self.add_array(key, value)+ def add_rope_pattern(self, value: Sequence[bool]) -> None:+ self.add_array(Keys.Attention.ROPE_PATTERN.format(arch=self.arch), value)+def add_dense_features_dims(self, dense:str, in_f:int, out_f:int) -> None:self.add_uint32(Keys.LLM.DENSE_FEAT_IN_SIZE.format(arch=self.arch, dense=dense), in_f)self.add_uint32(Keys.LLM.DENSE_FEAT_OUT_SIZE.format(arch=self.arch, dense=dense), out_f)diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex 3292942b41..a0571ccd32 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -458,6 +458,7 @@ class TensorNameMap:"transformer.decoder_layer.{bid}.router", # Grok"transformer.blocks.{bid}.ffn.router.layer", # dbrx"model.layers.{bid}.block_sparse_moe.router.layer", # granitemoe+ "model.layers.{bid}.block_sparse_moe.router", # granite_swa"model.layers.{bid}.feed_forward.router", # llama4 jamba"encoder.layers.{bid}.mlp.router.layer", # nomic-bert-moe"model.layers.{bid}.mlp.router", # openai-moediff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 5b88bde14d..955c2d7965 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -102,6 +102,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_GRANITE_MOE, "granitemoe" },{ LLM_ARCH_GRANITE_HYBRID, "granitehybrid" },{ LLM_ARCH_GRANITE_SWITCH, "graniteswitch" },+ { LLM_ARCH_GRANITE_SWA, "granite_swa" },{ LLM_ARCH_CHAMELEON, "chameleon" },{ LLM_ARCH_WAVTOKENIZER_DEC, "wavtokenizer-dec" },{ LLM_ARCH_PLM, "plm" },@@ -261,6 +262,8 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" },{ LLM_KV_ATTENTION_SLIDING_WINDOW, "%s.attention.sliding_window" },{ LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, "%s.attention.sliding_window_pattern" },+ { LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" },+{ LLM_KV_ATTENTION_SCALE, "%s.attention.scale" },{ LLM_KV_ATTENTION_OUTPUT_SCALE, "%s.attention.output_scale" },{ LLM_KV_ATTENTION_VALUE_SCALE, "%s.attention.value_scale" },diff --git a/src/llama-arch.h b/src/llama-arch.hindex 8042120a25..48fe051a99 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -107,6 +107,7 @@ enum llm_arch {LLM_ARCH_GRANITE_MOE,LLM_ARCH_GRANITE_HYBRID,LLM_ARCH_GRANITE_SWITCH,+ LLM_ARCH_GRANITE_SWA,LLM_ARCH_CHAMELEON,LLM_ARCH_WAVTOKENIZER_DEC,LLM_ARCH_PLM,@@ -267,6 +268,8 @@ enum llm_kv {LLM_KV_ATTENTION_SLIDING_WINDOW,LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN,LLM_KV_ATTENTION_SCALE,+ LLM_KV_ATTENTION_ROPE_PATTERN,+LLM_KV_ATTENTION_OUTPUT_SCALE,LLM_KV_ATTENTION_VALUE_SCALE,LLM_KV_ATTENTION_TEMPERATURE_LENGTH,diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cppindex e3f0cf0ede..cbe31134ff 100644--- a/src/llama-hparams.cpp+++ b/src/llama-hparams.cpp@@ -291,7 +291,11 @@ bool llama_hparams::has_rope(uint32_t il) const {return false;}- return true;+ if (il < n_layer_all) {+ return rope_pattern[il] != 0;+ }++ GGML_ABORT("%s: il (%u) out of bounds (n_layer_all: %u)\n", __func__, il, n_layer_all);}uint32_t llama_hparams::n_layer() const {diff --git a/src/llama-hparams.h b/src/llama-hparams.hindex e91ce1cc3c..f6af36436b 100644--- a/src/llama-hparams.h+++ b/src/llama-hparams.h@@ -144,6 +144,10 @@ struct llama_hparams {std::array<int, 4> rope_sections;+ // Per-layer RoPE enable flags (1 = use RoPE, 0 = NoPE)+ // by default, all layers use RoPE (controlled by rope_finetuned)+ std::array<uint32_t, LLAMA_MAX_LAYERS> rope_pattern;+// Sliding Window Attention (SWA)llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;// the size of the sliding window (0 - no SWA)diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cppindex be9524d404..b9e0a60094 100644--- a/src/llama-model-saver.cpp+++ b/src/llama-model-saver.cpp@@ -30,6 +30,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {case LLM_ARCH_MUSE_GLIMMER:case LLM_ARCH_MELLUM:case LLM_ARCH_LAGUNA:+ case LLM_ARCH_GRANITE_SWA:return false;default:return true;@@ -272,6 +273,7 @@ void llama_model_saver::add_kv_from_model() {add_kv(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix);add_kv(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate);add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);+ add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true);add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);// add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???);add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 0d74a2135b..3759c86259 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -246,6 +246,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_paramsreturn new llama_model_minicpm(params);case LLM_ARCH_GRANITE_HYBRID:return new llama_model_granite_hybrid(params);+ case LLM_ARCH_GRANITE_SWA:+ return new llama_model_granite_swa(params);case LLM_ARCH_CHAMELEON:return new llama_model_chameleon(params);case LLM_ARCH_WAVTOKENIZER_DEC:@@ -1157,6 +1159,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0);+ std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 1);std::fill(hparams.is_swa_impl.begin(), hparams.is_swa_impl.end(), 0);std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), llm_arch_is_recurrent(ml.get_arch()) ? 1 : 0);std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 0);@@ -2639,6 +2642,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {case LLM_ARCH_GRANITE_MOE:case LLM_ARCH_GRANITE_HYBRID:case LLM_ARCH_GRANITE_SWITCH:+ case LLM_ARCH_GRANITE_SWA:case LLM_ARCH_CHAMELEON:case LLM_ARCH_BAILINGMOE:case LLM_ARCH_BAILINGMOE3:diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cppindex eb23095aec..8a8f7e19ff 100644--- a/src/models/granite-hybrid.cpp+++ b/src/models/granite-hybrid.cpp@@ -16,7 +16,8 @@ void llama_model_granite_hybrid::load_arch_hparams(llama_model_loader & ml) {// Granite uses rope_finetuned as a switch for rope, so default to truebool rope_finetuned = true;ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);- hparams.rope_finetuned = rope_finetuned;+ hparams.rope_finetuned = rope_finetuned; // needed for round trip save+ std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned);// A layer is recurrent IFF the n_head_kv value is set to 0for (uint32_t i = 0; i < hparams.n_layer(); ++i) {@@ -147,7 +148,7 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr// Positional embeddings populated if rope enabledggml_tensor * inp_pos = nullptr;- if (hparams.rope_finetuned) {+ if (hparams.has_rope(0)) {inp_pos = build_inp_pos();}@@ -206,8 +207,7 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_attention_layer(ggml_tensconst int il) {auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);- const bool use_rope = hparams.rope_finetuned;- if (use_rope) {+ if (hparams.has_rope(il)) {ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);diff --git a/src/models/granite-moe.cpp b/src/models/granite-moe.cppindex 115263c418..09be49393e 100644--- a/src/models/granite-moe.cpp+++ b/src/models/granite-moe.cpp@@ -7,11 +7,6 @@ void llama_model_granite_moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false);ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false);- // Granite uses rope_finetuned as a switch for rope, so default to true- bool rope_finetuned = true;- ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);- hparams.rope_finetuned = rope_finetuned;-switch (hparams.n_layer()) {case 32: type = LLM_TYPE_3B; break;case 40: type = LLM_TYPE_3B; break;diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cppnew file mode 100644index 0000000000..3aa2b63b23--- /dev/null+++ b/src/models/granite-swa.cpp@@ -0,0 +1,319 @@+#include "models.h"++#include <sstream>++void llama_model_granite_swa::load_arch_hparams(llama_model_loader & ml) {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);+ ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale, false);+ ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false);+ ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale, false);++ // MoE expert configuration+ ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);+ ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);++ // iSWA configuration+ ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;++ // Granite4 Vision uses array deepstack_mapping+ ml.get_arr(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr, false);++ // Count the unique deepstack input indices+ std::unordered_set<uint32_t> unique_deepstack_idxs;+ for (const auto val : hparams.deepstack_mapping_arr) {+ if (val >= 0) {+ unique_deepstack_idxs.insert(val);+ }+ }+ hparams.n_deepstack_layers = unique_deepstack_idxs.size();++ // Ensure all values are valid (avoid overflow attacks)+ for (const auto val : unique_deepstack_idxs) {+ if (val > hparams.n_deepstack_layers) {+ std::stringstream ss;+ ss << "Invalid deepstack index: " << val << " > " << hparams.n_deepstack_layers;+ throw std::runtime_error(ss.str());+ }+ }++ // Per-layer RoPE pattern (optional)+ ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, false);++ switch (hparams.n_layer()) {+ case 32: type = LLM_TYPE_3B; break;+ case 40: type = LLM_TYPE_3B; break;+ // Add additional layer/vocab/etc checks here for other model sizes+ default: type = LLM_TYPE_UNKNOWN;+ }++ // For Granite MoE Shared+ ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, /* required */ false);+}++void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) {+ LLAMA_LOAD_LOCALS;++ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);++ // output+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);++ // if output is NULL, init from the input tok embed+ if (output == NULL) {+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);+ }++ for (int i = 0; i < n_layer; ++i) {+ auto & layer = layers[i];++ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);++ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);++ // optional bias tensors+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);++ // Per-layer attention sinks for iSWA+ layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0);++ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);++ if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {+ layer.rope_long = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));+ layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));+ }+ else {+ layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));+ }++ if (n_expert == 0) {+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);++ // optional MLP bias+ layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED);+ layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);+ layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, TENSOR_NOT_REQUIRED);+ } else {+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}, 0);+ create_tensor_gate_up_exps(layer, i, n_embd, n_ff, n_expert, 0);++ // For Granite MoE Shared - gate+up kept fused in ffn_up_shexp (see LLM_FFN_SWIGLU below)+ if (hparams.n_ff_shexp > 0) {+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, 2*hparams.n_ff_shexp}, 0);+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {hparams.n_ff_shexp, n_embd}, 0);+ }+ }+ }+}++std::unique_ptr<llm_graph_context> llama_model_granite_swa::build_arch_graph(const llm_graph_params & params) const {+ return std::make_unique<graph>(*this, params);+}++llama_model_granite_swa::graph::graph(+ const llama_model & model,+ const llm_graph_params & params)+ : llm_graph_context(params) {++ const int64_t n_embd_head = hparams.n_embd_head_v();++ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());+ GGML_ASSERT(n_embd_head == n_rot);++ ggml_tensor * cur;+ ggml_tensor * inpL;++ inpL = build_inp_embd(model.tok_embd);++ // inp_pos - built only if rope enabled+ ggml_tensor * inp_pos = build_inp_pos();+ auto * inp_attn = build_attn_inp_kv_iswa();++ ggml_tensor * inp_out_ids = build_inp_out_ids();++ for (int il = 0; il < n_layer; ++il) {++ // Granite Vision 4.1 deepstack: inject the projector stream that+ // targets decoder layer `il` before the decoder runs.+ // NOTE: skip the first deepstack layer since that's inpL+ const auto & deepstack_emb_idx = hparams.deepstack_mapping_arr[il];+ if (il > 0 && deepstack_emb_idx >= 0) {+ ggml_tensor * ds = ggml_view_2d(ctx0,+ res->t_inp_embd, n_embd, n_tokens,+ res->t_inp_embd->nb[1],+ deepstack_emb_idx * n_embd * sizeof(float));+ inpL = ggml_add(ctx0, inpL, ds);+ cb(inpL, "deepstack_in", il);+ }++ ggml_tensor * inpSA = inpL;++ // norm+ cur = build_norm(inpL,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, il);+ cb(cur, "attn_norm", il);++ // self-attention+ cur = build_attention_layer(+ cur, inp_pos, inp_attn,+ model, n_embd_head, il);++ if (il == n_layer - 1 && inp_out_ids) {+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+ }+ // ffn+ cur = build_layer_ffn(cur, inpSA, model, il);++ // input for next layer+ inpL = cur;+ }+ cur = inpL;++ cur = build_norm(cur,+ model.output_norm, NULL,+ LLM_NORM_RMS, -1);++ cb(cur, "result_norm", -1);+ res->t_embd = cur;++ // lm_head+ cur = build_lora_mm(model.output, cur, model.output_s);++ // For Granite architectures - scale logits+ cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale);+ cb(cur, "result_output", -1);+ res->t_logits = cur;++ ggml_build_forward_expand(gf, cur);+}++ggml_tensor * llama_model_granite_swa::graph::build_attention_layer(+ ggml_tensor * cur,+ ggml_tensor * inp_pos,+ llm_graph_input_attn_kv_iswa * inp_attn,+ const llama_model & model,+ const int64_t n_embd_head,+ const int il) {++ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,+ n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);++ const bool use_rope = hparams.has_rope(il);+ if (use_rope) {+ ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);+ Qcur = ggml_rope_ext(+ ctx0, Qcur, inp_pos, rope_factors,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );++ Kcur = ggml_rope_ext(+ ctx0, Kcur, inp_pos, rope_factors,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ }++ cb(Qcur, "Qcur", il);+ cb(Kcur, "Kcur", il);+ cb(Vcur, "Vcur", il);++ const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;++ // Pass layer.attn_sinks to build_attn for sink-based attention modulation+ cur = build_attn(inp_attn,+ model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,+ Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, kq_scale, il);+ cb(cur, "attn_out", il);+ return cur;+}++ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn(+ ggml_tensor * cur,+ ggml_tensor * inpSA,+ const llama_model & model,+ const int il) {++ // For Granite architectures - scale residual+ if (hparams.f_residual_scale) {+ cur = ggml_scale(ctx0, cur, hparams.f_residual_scale);+ }+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network (non-MoE)+ if (model.layers[il].ffn_gate_inp == nullptr) {++ cur = build_norm(ffn_inp,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, il);+ cb(cur, "ffn_norm", il);++ cur = build_ffn(cur,+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,+ model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, il);+ cb(cur, "ffn_out", il);++ } else {+ // MoE branch+ cur = build_norm(ffn_inp,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, il);+ cb(cur, "ffn_norm", il);++ ggml_tensor * moe_out = build_moe_ffn(cur,+ model.layers[il].ffn_gate_inp,+ model.layers[il].ffn_up_exps,+ model.layers[il].ffn_gate_exps,+ model.layers[il].ffn_down_exps,+ nullptr,+ n_expert, n_expert_used,+ LLM_FFN_SILU, true,+ hparams.expert_weights_scale,+ LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,+ il,+ nullptr, model.layers[il].ffn_gate_up_exps);+ cb(moe_out, "ffn_moe_out", il);++ // For Granite MoE Shared - gate+up kept fused in ffn_up_shexp+ if (hparams.n_ff_shexp > 0) {+ ggml_tensor * ffn_shexp = build_ffn(cur,+ model.layers[il].ffn_up_shexp, NULL, NULL,+ NULL, NULL, NULL,+ model.layers[il].ffn_down_shexp, NULL, NULL,+ NULL,+ LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);+ cb(ffn_shexp, "ffn_shexp", il);++ cur = ggml_add(ctx0, moe_out, ffn_shexp);+ cb(cur, "ffn_out", il);+ } else {+ cur = moe_out;+ }+ }++ // For Granite architectures - scale residual+ if (hparams.f_residual_scale) {+ cur = ggml_scale(ctx0, cur, hparams.f_residual_scale);+ }+ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "ffn_out", il);++ cur = build_cvec(cur, il);+ cb(cur, "l_out", il);++ return cur;+}diff --git a/src/models/granite-switch.cpp b/src/models/granite-switch.cppindex 80f6b86edc..7c9a901c8a 100644--- a/src/models/granite-switch.cpp+++ b/src/models/granite-switch.cpp@@ -11,7 +11,8 @@ void llama_model_granite_switch::load_arch_hparams(llama_model_loader & ml) {bool rope_finetuned = true;ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);- hparams.rope_finetuned = rope_finetuned;+ hparams.rope_finetuned = rope_finetuned; // needed for round trip save+ std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned);switch (hparams.n_layer()) {case 40: type = hparams.n_embd == 4096 ? LLM_TYPE_8B : LLM_TYPE_3B; break;@@ -254,7 +255,7 @@ llama_model_granite_switch::graph::graph(cb(inpL, "inp_embd", -1);ggml_tensor * inp_pos = nullptr;- if (hparams.rope_finetuned) {+ if (hparams.has_rope(0)) {inp_pos = build_inp_pos();}auto * inp_attn = build_attn_inp_kv();@@ -361,7 +362,7 @@ ggml_tensor * llama_model_granite_switch::graph::build_attention_layer(Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);- if (hparams.rope_finetuned) {+ if (hparams.has_rope(il)) {ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors,n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,diff --git a/src/models/granite.cpp b/src/models/granite.cppindex 4a75c5ff3c..9e9f97e94d 100644--- a/src/models/granite.cpp+++ b/src/models/granite.cpp@@ -33,7 +33,8 @@ void llama_model_granite::load_arch_hparams(llama_model_loader & ml) {// Granite uses rope_finetuned as a switch for rope, so default to truebool rope_finetuned = true;ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);- hparams.rope_finetuned = rope_finetuned;+ hparams.rope_finetuned = rope_finetuned; // needed for round trip save+ std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), rope_finetuned);switch (hparams.n_layer()) {case 32: type = LLM_TYPE_3B; break;@@ -127,7 +128,7 @@ llama_model_granite::graph::graph(// inp_pos - built only if rope enabledggml_tensor * inp_pos = nullptr;- if (hparams.rope_finetuned) {+ if (hparams.has_rope(0)) {inp_pos = build_inp_pos();}auto * inp_attn = build_attn_inp_kv();@@ -203,8 +204,7 @@ ggml_tensor * llama_model_granite::graph::build_attention_layer(auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);- const bool use_rope = hparams.rope_finetuned;- if (use_rope) {+ if (hparams.has_rope(il)) {ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors,diff --git a/src/models/models.h b/src/models/models.hindex 180b30a46d..1dd30dfd16 100644--- a/src/models/models.h+++ b/src/models/models.h@@ -1719,6 +1719,34 @@ struct llama_model_granite_hybrid : public llama_model_base {};+struct llama_model_granite_swa : public llama_model_base {+ llama_model_granite_swa(const struct llama_model_params & params) : llama_model_base(params) {}+ void load_arch_hparams(llama_model_loader & ml) override;+ void load_arch_tensors(llama_model_loader & ml) override;++ struct graph : public llm_graph_context {+ graph(const llama_model & model, const llm_graph_params & params);++ private:+ ggml_tensor * build_attention_layer(+ ggml_tensor * cur,+ ggml_tensor * inp_pos,+ llm_graph_input_attn_kv_iswa * inp_attn,+ const llama_model & model,+ const int64_t n_embd_head,+ const int il);++ ggml_tensor * build_layer_ffn(+ ggml_tensor * cur,+ ggml_tensor * inpSA,+ const llama_model & model,+ const int il);+ };++ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;+};++struct llama_model_chameleon : public llama_model_base {llama_model_chameleon(const struct llama_model_params & params) : llama_model_base(params) {}void load_arch_hparams(llama_model_loader & ml) override;diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex 032ac35376..4eb3763cbe 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -197,7 +197,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);// SWA pattern: every 5th layer is full attention (matches E2B layer_types)ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));- } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER) {+ } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA) {std::vector<uint32_t> pattern;pattern.reserve(n_layer);for (uint32_t il = 0; il < n_layer; il++) {