Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llama/parsers/renderers: nemotron 3 nano (#13489)
Parth Sareen7e3ea813c1d8
diff --git a/llama/llama.cpp/src/llama-arch.cpp b/llama/llama.cpp/src/llama-arch.cppindex a5fe4f66c..ac8b5e033 100644--- a/llama/llama.cpp/src/llama-arch.cpp+++ b/llama/llama.cpp/src/llama-arch.cpp@@ -75,6 +75,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_JAIS, "jais" },{ LLM_ARCH_NEMOTRON, "nemotron" },{ LLM_ARCH_NEMOTRON_H, "nemotron_h" },+ { LLM_ARCH_NEMOTRON_H_MOE, "nemotron_h_moe" },{ LLM_ARCH_EXAONE, "exaone" },{ LLM_ARCH_EXAONE4, "exaone4" },{ LLM_ARCH_RWKV6, "rwkv6" },@@ -1765,6 +1766,39 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_NEMOTRON_H_MOE,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ // mamba(2) ssm layers+ { LLM_TENSOR_SSM_IN, "blk.%d.ssm_in" },+ { LLM_TENSOR_SSM_CONV1D, "blk.%d.ssm_conv1d" },+ { LLM_TENSOR_SSM_DT, "blk.%d.ssm_dt" },+ { LLM_TENSOR_SSM_A, "blk.%d.ssm_a" },+ { LLM_TENSOR_SSM_D, "blk.%d.ssm_d" },+ { LLM_TENSOR_SSM_NORM, "blk.%d.ssm_norm" },+ { LLM_TENSOR_SSM_OUT, "blk.%d.ssm_out" },+ // attention layers+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ // dense FFN+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ // MoE FFN (for MoE layers)+ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },+ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },+ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },+ { LLM_TENSOR_FFN_EXP_PROBS_B,"blk.%d.exp_probs_b" },+ // MoE shared expert layer+ { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" },+ { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" },+ },+ },{LLM_ARCH_EXAONE,{@@ -2838,6 +2872,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {case LLM_ARCH_LFM2:case LLM_ARCH_LFM2MOE:case LLM_ARCH_NEMOTRON_H:+ case LLM_ARCH_NEMOTRON_H_MOE:case LLM_ARCH_QWEN3NEXT:return true;default:diff --git a/llama/llama.cpp/src/llama-arch.h b/llama/llama.cpp/src/llama-arch.hindex ec9e3a6df..61d73786c 100644--- a/llama/llama.cpp/src/llama-arch.h+++ b/llama/llama.cpp/src/llama-arch.h@@ -79,6 +79,7 @@ enum llm_arch {LLM_ARCH_JAIS,LLM_ARCH_NEMOTRON,LLM_ARCH_NEMOTRON_H,+ LLM_ARCH_NEMOTRON_H_MOE,LLM_ARCH_EXAONE,LLM_ARCH_EXAONE4,LLM_ARCH_RWKV6,diff --git a/llama/llama.cpp/src/llama-graph.cpp b/llama/llama.cpp/src/llama-graph.cppindex 43620df78..763202d87 100644--- a/llama/llama.cpp/src/llama-graph.cpp+++ b/llama/llama.cpp/src/llama-graph.cpp@@ -1089,6 +1089,16 @@ ggml_tensor * llm_graph_context::build_moe_ffn(cur = ggml_relu(ctx0, cur);cb(cur, "ffn_moe_relu", il);} break;+ case LLM_FFN_RELU_SQR:+ if (gate_exps) {+ // TODO: add support for gated squared relu+ GGML_ABORT("fatal error: gated squared relu not implemented");+ } else {+ cur = ggml_relu(ctx0, cur);+ cur = ggml_sqr(ctx0, cur);+ cb(cur, "ffn_moe_relu_sqr", il);+ }+ break;default:GGML_ABORT("fatal error");}diff --git a/llama/llama.cpp/src/llama-model.cpp b/llama/llama.cpp/src/llama-model.cppindex 3c503b424..94dee78c3 100644--- a/llama/llama.cpp/src/llama-model.cpp+++ b/llama/llama.cpp/src/llama-model.cpp@@ -120,6 +120,8 @@ const char * llm_type_name(llm_type type) {case LLM_TYPE_16B_A1B: return "16B.A1B";case LLM_TYPE_21B_A3B: return "21B.A3B";case LLM_TYPE_30B_A3B: return "30B.A3B";+ case LLM_TYPE_31B_A3_5B: return "31B.A3.5B";+ case LLM_TYPE_80B_A3B: return "80B.A3B";case LLM_TYPE_100B_A6B: return "100B.A6B";case LLM_TYPE_106B_A12B: return "106B.A12B";case LLM_TYPE_230B_A10B: return "230B.A10B";@@ -1788,6 +1790,7 @@ void llama_model::load_hparams(llama_model_loader & ml) {}} break;case LLM_ARCH_NEMOTRON_H:+ case LLM_ARCH_NEMOTRON_H_MOE:{ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);@@ -1803,7 +1806,14 @@ void llama_model::load_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);+ ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);+ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false);+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);+switch (hparams.n_layer) {+ case 52: type = LLM_TYPE_31B_A3_5B; break; // Nemotron-H_MOE 31Bcase 56: type = LLM_TYPE_9B; break;default: type = LLM_TYPE_UNKNOWN;}@@ -5175,6 +5185,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {}} break;case LLM_ARCH_NEMOTRON_H:+ case LLM_ARCH_NEMOTRON_H_MOE:{// mamba2 Mixer SSM params// NOTE: int64_t for tensor dimensions@@ -5185,6 +5196,9 @@ bool llama_model::load_tensors(llama_model_loader & ml) {const int64_t n_group = hparams.ssm_n_group;const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_ssm_head;+ const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;+ const int64_t n_ff_shexp = hparams.n_ff_shexp;+// embeddingstok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);@@ -5234,12 +5248,26 @@ bool llama_model::load_tensors(llama_model_loader & ml) {layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_k_gqa_i}, TENSOR_NOT_REQUIRED);layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_v_gqa_i}, TENSOR_NOT_REQUIRED);layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);- } else {- // mlp layers- layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { hparams.n_ff(i), n_embd}, 0);- layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, hparams.n_ff(i)}, 0);- layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);- layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {hparams.n_ff(i)}, TENSOR_NOT_REQUIRED);+ } else {+ if (n_expert != 0) {+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, 0);+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert }, 0);++ // MoE branch+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);++ // Shared expert branch+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0);+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0);++ } else {+ // mlp layers+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { hparams.n_ff(i), n_embd}, 0);+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, hparams.n_ff(i)}, 0);+ layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);+ layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {hparams.n_ff(i)}, TENSOR_NOT_REQUIRED);+ }}}} break;@@ -6870,7 +6898,8 @@ void llama_model::print_info() const {arch == LLM_ARCH_PLAMO2 ||arch == LLM_ARCH_GRANITE_HYBRID ||arch == LLM_ARCH_QWEN3NEXT ||- arch == LLM_ARCH_NEMOTRON_H) {+ arch == LLM_ARCH_NEMOTRON_H ||+ arch == LLM_ARCH_NEMOTRON_H_MOE) {LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state);@@ -6926,7 +6955,8 @@ void llama_model::print_info() const {if (arch == LLM_ARCH_MINICPM ||arch == LLM_ARCH_GRANITE ||arch == LLM_ARCH_GRANITE_MOE ||- arch == LLM_ARCH_GRANITE_HYBRID) {+ arch == LLM_ARCH_GRANITE_HYBRID ||+ arch == LLM_ARCH_NEMOTRON_H_MOE) {LLAMA_LOG_INFO("%s: f_embedding_scale = %f\n", __func__, hparams.f_embedding_scale);LLAMA_LOG_INFO("%s: f_residual_scale = %f\n", __func__, hparams.f_residual_scale);LLAMA_LOG_INFO("%s: f_attention_scale = %f\n", __func__, hparams.f_attention_scale);@@ -7107,7 +7137,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,if (arch == LLM_ARCH_FALCON_H1) {filter_attn = [&](int32_t) { return true; };filter_recr = [&](int32_t) { return true; };- } else if (arch == LLM_ARCH_NEMOTRON_H) {+ } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {filter_attn = [&](int32_t il) {return !hparams.is_recurrent(il) && hparams.n_ff(il) == 0;};@@ -7478,6 +7508,7 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {llm = std::make_unique<llm_build_nemotron>(*this, params);} break;case LLM_ARCH_NEMOTRON_H:+ case LLM_ARCH_NEMOTRON_H_MOE:{llm = std::make_unique<llm_build_nemotron_h>(*this, params);} break;@@ -7765,6 +7796,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {case LLM_ARCH_ARWKV7:case LLM_ARCH_WAVTOKENIZER_DEC:case LLM_ARCH_NEMOTRON_H:+ case LLM_ARCH_NEMOTRON_H_MOE:return LLAMA_ROPE_TYPE_NONE;// use what we call a normal RoPE, operating on pairs of consecutive head valuesdiff --git a/llama/llama.cpp/src/llama-model.h b/llama/llama.cpp/src/llama-model.hindex cbf4e1bfa..b378b23ec 100644--- a/llama/llama.cpp/src/llama-model.h+++ b/llama/llama.cpp/src/llama-model.h@@ -114,6 +114,7 @@ enum llm_type {LLM_TYPE_16B_A1B,LLM_TYPE_21B_A3B, // Ernie MoE smallLLM_TYPE_30B_A3B,+ LLM_TYPE_31B_A3_5B,LLM_TYPE_80B_A3B, // Qwen3 NextLLM_TYPE_100B_A6B,LLM_TYPE_106B_A12B, // GLM-4.5-Airdiff --git a/llama/llama.cpp/src/models/nemotron-h.cpp b/llama/llama.cpp/src/models/nemotron-h.cppindex 541434888..eb135e63f 100644--- a/llama/llama.cpp/src/models/nemotron-h.cpp+++ b/llama/llama.cpp/src/models/nemotron-h.cpp@@ -107,12 +107,41 @@ ggml_tensor * llm_build_nemotron_h::build_attention_layer(ggml_tensor *}ggml_tensor * llm_build_nemotron_h::build_ffn_layer(ggml_tensor * cur, const llama_model & model, const int il) {- cur = build_ffn(cur,- model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,- NULL, NULL, NULL,- model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,- NULL, LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);- cb(cur, "ffn_out", il);+ if (model.layers[il].ffn_gate_inp == nullptr) {+ cur = build_ffn(cur,+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,+ NULL, NULL, NULL,+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,+ NULL,+ LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);+ cb(cur, "ffn_out", il);+ } else {+ ggml_tensor * ffn_inp = cur;+ ggml_tensor * moe_out =+ build_moe_ffn(ffn_inp,+ model.layers[il].ffn_gate_inp,+ model.layers[il].ffn_up_exps,+ nullptr, // no gate+ model.layers[il].ffn_down_exps,+ model.layers[il].ffn_exp_probs_b,+ n_expert, n_expert_used,+ LLM_FFN_RELU_SQR, hparams.expert_weights_norm,+ true, hparams.expert_weights_scale,+ LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,+ il);+ cb(moe_out, "ffn_moe_out", il);++ ggml_tensor * ffn_shexp = build_ffn(ffn_inp,+ model.layers[il].ffn_up_shexp, NULL, NULL,+ NULL /* no gate */ , NULL, NULL,+ model.layers[il].ffn_down_shexp, NULL, NULL,+ NULL,+ LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);+ cb(ffn_shexp, "ffn_shexp", il);++ cur = ggml_add(ctx0, moe_out, ffn_shexp);+ cb(cur, "ffn_out", il);+ }cur = build_cvec(cur, il);cb(cur, "l_out", il);diff --git a/llama/patches/0032-llama-add-support-for-NVIDIA-Nemotron-Nano-3.patch b/llama/patches/0032-llama-add-support-for-NVIDIA-Nemotron-Nano-3.patchnew file mode 100644index 000000000..00536c4b8--- /dev/null+++ b/llama/patches/0032-llama-add-support-for-NVIDIA-Nemotron-Nano-3.patch@@ -0,0 +1,586 @@+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001+From: Daniel Bevenius <[email protected]>+Date: Mon, 15 Dec 2025 15:13:49 +0100+Subject: [PATCH] llama : add support for NVIDIA Nemotron Nano 3++This commit adds support for the NVIDIA Nemotron Nano 3 model, enabling+the conversion and running of this model.++fix indentation in llama-graph.cpp++fix indentation and move ffn_inp++convert : fix modify_tensors in NemotronHModel to call super()++fix pyright error++fix flake8 errors+---+ convert_hf_to_gguf.py | 116 +++++++++++++++++++++++++++++++--+ gguf-py/gguf/constants.py | 29 ++++++++++ gguf-py/gguf/tensor_mapping.py | 9 ++-+ src/llama-arch.cpp | 35 +++++++++++ src/llama-arch.h | 1 ++ src/llama-graph.cpp | 10 ++++ src/llama-model.cpp | 50 +++++++++++---+ src/llama-model.h | 1 ++ src/models/nemotron-h.cpp | 41 ++++++++++--+ 9 files changed, 269 insertions(+), 23 deletions(-)++diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py+index 867bc9053..57ec2faac 100755+--- a/convert_hf_to_gguf.py++++ b/convert_hf_to_gguf.py+@@ -8601,8 +8601,18 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):+ class NemotronHModel(GraniteHybridModel):+ """Hybrid mamba2/attention model from NVIDIA"""+ model_arch = gguf.MODEL_ARCH.NEMOTRON_H++ is_moe: bool = False++ def __init__(self, *args, **kwargs):++ # We have to determine the correct model architecture (MoE vs non-MoE) before++ # calling the parent __init__. This is because the parent constructor++ # uses self.model_arch to build the tensor name map, and all MoE-specific++ # mappings would be missed if it were called with the default non-MoE arch.++ hparams = ModelBase.load_hparams(args[0], self.is_mistral_format)++ if "num_experts_per_tok" in hparams:++ self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE++ self.is_moe = True+++ super().__init__(*args, **kwargs)++ # Save the top-level head_dim for later+@@ -8614,9 +8624,11 @@ class NemotronHModel(GraniteHybridModel):++ # Update the ssm / attn / mlp layers+ # M: Mamba2, *: Attention, -: MLP++ # MoE:++ # M: Mamba2, *: Attention, E: Expert+ hybrid_override_pattern = self.hparams["hybrid_override_pattern"]+ self._ssm_layers = [i for i, val in enumerate(hybrid_override_pattern) if val == "M"]+- self._mlp_layers = [i for i, val in enumerate(hybrid_override_pattern) if val == "-"]++ self._mlp_layers = [i for i, val in enumerate(hybrid_override_pattern) if val == ("E" if self.is_moe else "-")]++ def get_attn_layers(self):+ hybrid_override_pattern = self.hparams["hybrid_override_pattern"]+@@ -8632,10 +8644,28 @@ class NemotronHModel(GraniteHybridModel):+ # Set feed_forward_length+ # NOTE: This will trigger an override warning. This is preferrable to+ # duplicating all the parent logic+- n_ff = self.find_hparam(["intermediate_size", "n_inner", "hidden_dim"])+- self.gguf_writer.add_feed_forward_length([+- n_ff if i in self._mlp_layers else 0 for i in range(self.block_count)+- ])++ if not self.is_moe:++ n_ff = self.find_hparam(["intermediate_size", "n_inner", "hidden_dim"])++ self.gguf_writer.add_feed_forward_length([++ n_ff if i in self._mlp_layers else 0 for i in range(self.block_count)++ ])++ else:++ moe_intermediate_size = self.hparams["moe_intermediate_size"]++ self.gguf_writer.add_feed_forward_length([++ moe_intermediate_size if i in self._mlp_layers else 0 for i in range(self.block_count)++ ])++ self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])++ self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])++ self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])++ self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"])++ self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])++ self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])++ self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])++ self.gguf_writer.add_expert_group_count(self.hparams["n_group"])++++ # number of experts used per token (top-k)++ if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:++ self.gguf_writer.add_expert_used_count(n_experts_used)++ def set_vocab(self):+ super().set_vocab()+@@ -8643,7 +8673,81 @@ class NemotronHModel(GraniteHybridModel):+ # The tokenizer _does_ add a BOS token (via post_processor type+ # TemplateProcessing) but does not set add_bos_token to true in the+ # config, so we need to explicitly override it here.+- self.gguf_writer.add_add_bos_token(True)++ if not self.is_moe:++ self.gguf_writer.add_add_bos_token(True)++++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:++ if self.is_moe and bid is not None:++ if name.endswith("mixer.gate.e_score_correction_bias"):++ new_name = name.replace("e_score_correction_bias", "e_score_correction_bias.bias")++ mapped_name = self.map_tensor_name(new_name)++ return [(mapped_name, data_torch)]++++ if name.endswith("mixer.dt_bias"):++ new_name = name.replace("dt_bias", "dt.bias")++ mapped_name = self.map_tensor_name(new_name)++ return [(mapped_name, data_torch)]++++ if name.endswith("mixer.conv1d.weight"):++ squeezed_data = data_torch.squeeze()++ mapped_name = self.map_tensor_name(name)++ return [(mapped_name, squeezed_data)]++++ if name.endswith("mixer.A_log"):++ transformed_data = -torch.exp(data_torch)++ reshaped_data = transformed_data.squeeze().reshape(-1, 1)++ mapped_name = self.map_tensor_name(name)++ return [(mapped_name, reshaped_data)]++++ if name.endswith("mixer.D"):++ reshaped_data = data_torch.squeeze().reshape(-1, 1)++ mapped_name = self.map_tensor_name(name)++ return [(mapped_name, reshaped_data)]++++ if name.endswith("mixer.norm.weight"):++ reshaped_data = data_torch.reshape(8, 512)++ mapped_name = self.map_tensor_name(name)++ return [(mapped_name, reshaped_data)]++++ if name.find("mixer.experts") != -1:++ n_experts = self.hparams["n_routed_experts"]++ assert bid is not None++++ if self._experts is None:++ self._experts = [{} for _ in range(self.block_count)]++++ self._experts[bid][name] = data_torch++++ if len(self._experts[bid]) >= n_experts * 2:++ # merge the experts into a single tensor++ tensors: list[tuple[str, Tensor]] = []++ for w_name in ["down_proj", "up_proj"]:++ datas: list[Tensor] = []++++ for xid in range(n_experts):++ ename = f"backbone.layers.{bid}.mixer.experts.{xid}.{w_name}.weight"++ datas.append(self._experts[bid][ename])++ del self._experts[bid][ename]++++ data_torch = torch.stack(datas, dim=0)++ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"++ new_name = self.map_tensor_name(merged_name)++ tensors.append((new_name, data_torch))++++ return tensors++ else:++ return []++++ return super().modify_tensors(data_torch, name, bid)++++ def prepare_tensors(self):++ super().prepare_tensors()++++ if self._experts is not None:++ # flatten `list[dict[str, Tensor]]` into `list[str]`++ experts = [k for d in self._experts for k in d.keys()]++ if len(experts) > 0:++ raise ValueError(f"Unprocessed experts: {experts}")+++ @ModelBase.register("BailingMoeForCausalLM")+diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py+index 2b8489c59..1852428b4 100644+--- a/gguf-py/gguf/constants.py++++ b/gguf-py/gguf/constants.py+@@ -413,6 +413,7 @@ class MODEL_ARCH(IntEnum):+ JAIS = auto()+ NEMOTRON = auto()+ NEMOTRON_H = auto()++ NEMOTRON_H_MOE = auto()+ EXAONE = auto()+ EXAONE4 = auto()+ GRANITE = auto()+@@ -786,6 +787,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {+ MODEL_ARCH.JAIS: "jais",+ MODEL_ARCH.NEMOTRON: "nemotron",+ MODEL_ARCH.NEMOTRON_H: "nemotron_h",++ MODEL_ARCH.NEMOTRON_H_MOE: "nemotron_h_moe",+ MODEL_ARCH.EXAONE: "exaone",+ MODEL_ARCH.EXAONE4: "exaone4",+ MODEL_ARCH.GRANITE: "granite",+@@ -2529,6 +2531,33 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],++ MODEL_ARCH.NEMOTRON_H_MOE: [++ MODEL_TENSOR.TOKEN_EMBD,++ MODEL_TENSOR.OUTPUT_NORM,++ MODEL_TENSOR.OUTPUT,++ MODEL_TENSOR.ATTN_NORM,++ MODEL_TENSOR.SSM_IN,++ MODEL_TENSOR.SSM_CONV1D,++ MODEL_TENSOR.SSM_DT,++ MODEL_TENSOR.SSM_A,++ MODEL_TENSOR.SSM_D,++ MODEL_TENSOR.SSM_NORM,++ MODEL_TENSOR.SSM_OUT,++ MODEL_TENSOR.ATTN_Q,++ MODEL_TENSOR.ATTN_K,++ MODEL_TENSOR.ATTN_V,++ MODEL_TENSOR.ATTN_OUT,++ MODEL_TENSOR.FFN_DOWN,++ MODEL_TENSOR.FFN_UP,++ # experts++ MODEL_TENSOR.FFN_GATE_INP,++ MODEL_TENSOR.FFN_UP_EXP,++ MODEL_TENSOR.FFN_DOWN_EXP,++ # shared expert++ MODEL_TENSOR.FFN_DOWN_SHEXP,++ MODEL_TENSOR.FFN_UP_SHEXP,++ MODEL_TENSOR.FFN_EXP_PROBS_B,++ ],+ MODEL_ARCH.EXAONE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py+index d9c87da19..7a3c7c5e0 100644+--- a/gguf-py/gguf/tensor_mapping.py++++ b/gguf-py/gguf/tensor_mapping.py+@@ -377,6 +377,7 @@ class TensorNameMap:+ "model.layers.{bid}.feed_forward.gate", # lfm2moe+ "model.layers.{bid}.mlp.router.gate", # afmoe+ "layers.{bid}.gate", # mistral-large++ "backbone.layers.{bid}.mixer.gate", # nemotron-h-moe+ ),++ MODEL_TENSOR.FFN_GATE_INP_SHEXP: (+@@ -390,6 +391,7 @@ class TensorNameMap:+ "model.layers.{bid}.mlp.expert_bias", # afmoe+ "model.layers.{bid}.feed_forward.expert_bias", # lfm2moe+ "model.layers.{bid}.block_sparse_moe.e_score_correction", # minimax-m2++ "backbone.layers.{bid}.mixer.gate.e_score_correction_bias" # nemotron-h-moe+ ),++ # Feed-forward up+@@ -438,7 +440,7 @@ class TensorNameMap:+ "layers.{bid}.feed_forward.experts.w3", # mixtral (merged)+ "transformer.decoder_layer.{bid}.moe.linear_v", # Grok (merged)+ "transformer.blocks.{bid}.ffn.experts.mlp.v1", # dbrx+- "model.layers.{bid}.mlp.experts.up_proj", # qwen2moe olmoe (merged) ernie4.5-moe++ "model.layers.{bid}.mlp.experts.up_proj", # qwen2moe olmoe (merged) ernie4.5-moe, nemotron-h-moe (merged)+ "model.layers.{bid}.block_sparse_moe.experts.w3", # phimoe (merged)+ "model.layers.{bid}.feed_forward.experts.up_proj", # llama4+ "encoder.layers.{bid}.mlp.experts.mlp.w1", # nomic-bert-moe+@@ -452,6 +454,7 @@ class TensorNameMap:+ "model.layers.{bid}.feed_forward.down_proj",+ "model.layers.{bid}.mlp.shared_mlp.up_proj", # hunyuan+ "layers.{bid}.shared_experts.w3", # mistral-large++ "backbone.layers.{bid}.mixer.shared_experts.up_proj", # nemotron-h-moe+ ),++ MODEL_TENSOR.FFN_UP_CHEXP: (+@@ -546,7 +549,7 @@ class TensorNameMap:+ "layers.{bid}.feed_forward.experts.w2", # mixtral (merged)+ "transformer.decoder_layer.{bid}.moe.linear_1", # Grok (merged)+ "transformer.blocks.{bid}.ffn.experts.mlp.w2", # dbrx+- "model.layers.{bid}.mlp.experts.down_proj", # qwen2moe olmoe (merged) ernie4.5-moe++ "model.layers.{bid}.mlp.experts.down_proj", # qwen2moe olmoe (merged) ernie4.5-moe nemotron-h-moe (merged)+ "model.layers.{bid}.block_sparse_moe.output_linear", # granitemoe+ "model.layers.{bid}.block_sparse_moe.experts.w2", # phimoe (merged)+ "model.layers.{bid}.feed_forward.experts.down_proj", # llama4+@@ -561,6 +564,7 @@ class TensorNameMap:+ "model.layers.{bid}.shared_mlp.output_linear", # granitemoe+ "model.layers.{bid}.mlp.shared_mlp.down_proj", # hunyuan+ "layers.{bid}.shared_experts.w2", # mistral-large++ "backbone.layers.{bid}.mixer.shared_experts.down_proj", # nemotron-h-moe+ ),++ MODEL_TENSOR.FFN_DOWN_CHEXP: (+@@ -704,6 +708,7 @@ class TensorNameMap:+ "model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid+ "model.layers.layers.{bid}.mixer.dt_proj", # plamo2+ "model.layers.{bid}.linear_attn.dt_proj", # qwen3next++ "backbone.layers.{bid}.mixer.dt", # nemotron-h-moe+ ),++ MODEL_TENSOR.SSM_DT_NORM: (+diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp+index a5fe4f66c..ac8b5e033 100644+--- a/src/llama-arch.cpp++++ b/src/llama-arch.cpp+@@ -75,6 +75,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {+ { LLM_ARCH_JAIS, "jais" },+ { LLM_ARCH_NEMOTRON, "nemotron" },+ { LLM_ARCH_NEMOTRON_H, "nemotron_h" },++ { LLM_ARCH_NEMOTRON_H_MOE, "nemotron_h_moe" },+ { LLM_ARCH_EXAONE, "exaone" },+ { LLM_ARCH_EXAONE4, "exaone4" },+ { LLM_ARCH_RWKV6, "rwkv6" },+@@ -1765,6 +1766,39 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },++ {++ LLM_ARCH_NEMOTRON_H_MOE,++ {++ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },++ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },++ { LLM_TENSOR_OUTPUT, "output" },++ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },++ // mamba(2) ssm layers++ { LLM_TENSOR_SSM_IN, "blk.%d.ssm_in" },++ { LLM_TENSOR_SSM_CONV1D, "blk.%d.ssm_conv1d" },++ { LLM_TENSOR_SSM_DT, "blk.%d.ssm_dt" },++ { LLM_TENSOR_SSM_A, "blk.%d.ssm_a" },++ { LLM_TENSOR_SSM_D, "blk.%d.ssm_d" },++ { LLM_TENSOR_SSM_NORM, "blk.%d.ssm_norm" },++ { LLM_TENSOR_SSM_OUT, "blk.%d.ssm_out" },++ // attention layers++ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },++ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },++ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },++ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },++ // dense FFN++ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },++ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },++ // MoE FFN (for MoE layers)++ { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },++ { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },++ { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },++ { LLM_TENSOR_FFN_EXP_PROBS_B,"blk.%d.exp_probs_b" },++ // MoE shared expert layer++ { LLM_TENSOR_FFN_DOWN_SHEXP, "blk.%d.ffn_down_shexp" },++ { LLM_TENSOR_FFN_UP_SHEXP, "blk.%d.ffn_up_shexp" },++ },++ },+ {+ LLM_ARCH_EXAONE,+ {+@@ -2838,6 +2872,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {+ case LLM_ARCH_LFM2:+ case LLM_ARCH_LFM2MOE:+ case LLM_ARCH_NEMOTRON_H:++ case LLM_ARCH_NEMOTRON_H_MOE:+ case LLM_ARCH_QWEN3NEXT:+ return true;+ default:+diff --git a/src/llama-arch.h b/src/llama-arch.h+index ec9e3a6df..61d73786c 100644+--- a/src/llama-arch.h++++ b/src/llama-arch.h+@@ -79,6 +79,7 @@ enum llm_arch {+ LLM_ARCH_JAIS,+ LLM_ARCH_NEMOTRON,+ LLM_ARCH_NEMOTRON_H,++ LLM_ARCH_NEMOTRON_H_MOE,+ LLM_ARCH_EXAONE,+ LLM_ARCH_EXAONE4,+ LLM_ARCH_RWKV6,+diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp+index 43620df78..763202d87 100644+--- a/src/llama-graph.cpp++++ b/src/llama-graph.cpp+@@ -1089,6 +1089,16 @@ ggml_tensor * llm_graph_context::build_moe_ffn(+ cur = ggml_relu(ctx0, cur);+ cb(cur, "ffn_moe_relu", il);+ } break;++ case LLM_FFN_RELU_SQR:++ if (gate_exps) {++ // TODO: add support for gated squared relu++ GGML_ABORT("fatal error: gated squared relu not implemented");++ } else {++ cur = ggml_relu(ctx0, cur);++ cur = ggml_sqr(ctx0, cur);++ cb(cur, "ffn_moe_relu_sqr", il);++ }++ break;+ default:+ GGML_ABORT("fatal error");+ }+diff --git a/src/llama-model.cpp b/src/llama-model.cpp+index 3c503b424..94dee78c3 100644+--- a/src/llama-model.cpp++++ b/src/llama-model.cpp+@@ -120,6 +120,8 @@ const char * llm_type_name(llm_type type) {+ case LLM_TYPE_16B_A1B: return "16B.A1B";+ case LLM_TYPE_21B_A3B: return "21B.A3B";+ case LLM_TYPE_30B_A3B: return "30B.A3B";++ case LLM_TYPE_31B_A3_5B: return "31B.A3.5B";++ case LLM_TYPE_80B_A3B: return "80B.A3B";+ case LLM_TYPE_100B_A6B: return "100B.A6B";+ case LLM_TYPE_106B_A12B: return "106B.A12B";+ case LLM_TYPE_230B_A10B: return "230B.A10B";+@@ -1788,6 +1790,7 @@ void llama_model::load_hparams(llama_model_loader & ml) {+ }+ } break;+ case LLM_ARCH_NEMOTRON_H:++ case LLM_ARCH_NEMOTRON_H_MOE:+ {+ ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);+ ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);+@@ -1803,7 +1806,14 @@ void llama_model::load_hparams(llama_model_loader & ml) {++ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+++ ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);++ ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);++ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false);++ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);++ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);+++ switch (hparams.n_layer) {++ case 52: type = LLM_TYPE_31B_A3_5B; break; // Nemotron-H_MOE 31B+ case 56: type = LLM_TYPE_9B; break;+ default: type = LLM_TYPE_UNKNOWN;+ }+@@ -5175,6 +5185,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {+ }+ } break;+ case LLM_ARCH_NEMOTRON_H:++ case LLM_ARCH_NEMOTRON_H_MOE:+ {+ // mamba2 Mixer SSM params+ // NOTE: int64_t for tensor dimensions+@@ -5185,6 +5196,9 @@ bool llama_model::load_tensors(llama_model_loader & ml) {+ const int64_t n_group = hparams.ssm_n_group;+ const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_ssm_head;+++ const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;++ const int64_t n_ff_shexp = hparams.n_ff_shexp;+++ // embeddings+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);++@@ -5234,12 +5248,26 @@ bool llama_model::load_tensors(llama_model_loader & ml) {+ layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_k_gqa_i}, TENSOR_NOT_REQUIRED);+ layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_v_gqa_i}, TENSOR_NOT_REQUIRED);+ layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);+- } else {+- // mlp layers+- layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { hparams.n_ff(i), n_embd}, 0);+- layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, hparams.n_ff(i)}, 0);+- layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);+- layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {hparams.n_ff(i)}, TENSOR_NOT_REQUIRED);++ } else {++ if (n_expert != 0) {++ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, 0);++ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert }, 0);++++ // MoE branch++ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);++ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);++++ // Shared expert branch++ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0);++ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0);++++ } else {++ // mlp layers++ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { hparams.n_ff(i), n_embd}, 0);++ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, hparams.n_ff(i)}, 0);++ layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);++ layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {hparams.n_ff(i)}, TENSOR_NOT_REQUIRED);++ }+ }+ }+ } break;+@@ -6870,7 +6898,8 @@ void llama_model::print_info() const {+ arch == LLM_ARCH_PLAMO2 ||+ arch == LLM_ARCH_GRANITE_HYBRID ||+ arch == LLM_ARCH_QWEN3NEXT ||+- arch == LLM_ARCH_NEMOTRON_H) {++ arch == LLM_ARCH_NEMOTRON_H ||++ arch == LLM_ARCH_NEMOTRON_H_MOE) {+ LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);+ LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);+ LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state);+@@ -6926,7 +6955,8 @@ void llama_model::print_info() const {+ if (arch == LLM_ARCH_MINICPM ||+ arch == LLM_ARCH_GRANITE ||+ arch == LLM_ARCH_GRANITE_MOE ||+- arch == LLM_ARCH_GRANITE_HYBRID) {++ arch == LLM_ARCH_GRANITE_HYBRID ||++ arch == LLM_ARCH_NEMOTRON_H_MOE) {+ LLAMA_LOG_INFO("%s: f_embedding_scale = %f\n", __func__, hparams.f_embedding_scale);+ LLAMA_LOG_INFO("%s: f_residual_scale = %f\n", __func__, hparams.f_residual_scale);+ LLAMA_LOG_INFO("%s: f_attention_scale = %f\n", __func__, hparams.f_attention_scale);+@@ -7107,7 +7137,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,+ if (arch == LLM_ARCH_FALCON_H1) {+ filter_attn = [&](int32_t) { return true; };+ filter_recr = [&](int32_t) { return true; };+- } else if (arch == LLM_ARCH_NEMOTRON_H) {++ } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {+ filter_attn = [&](int32_t il) {+ return !hparams.is_recurrent(il) && hparams.n_ff(il) == 0;+ };+@@ -7478,6 +7508,7 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {+ llm = std::make_unique<llm_build_nemotron>(*this, params);+ } break;+ case LLM_ARCH_NEMOTRON_H:++ case LLM_ARCH_NEMOTRON_H_MOE:+ {+ llm = std::make_unique<llm_build_nemotron_h>(*this, params);+ } break;+@@ -7765,6 +7796,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {+ case LLM_ARCH_ARWKV7:+ case LLM_ARCH_WAVTOKENIZER_DEC:+ case LLM_ARCH_NEMOTRON_H:++ case LLM_ARCH_NEMOTRON_H_MOE:+ return LLAMA_ROPE_TYPE_NONE;++ // use what we call a normal RoPE, operating on pairs of consecutive head values+diff --git a/src/llama-model.h b/src/llama-model.h+index cbf4e1bfa..b378b23ec 100644+--- a/src/llama-model.h++++ b/src/llama-model.h+@@ -114,6 +114,7 @@ enum llm_type {+ LLM_TYPE_16B_A1B,+ LLM_TYPE_21B_A3B, // Ernie MoE small+ LLM_TYPE_30B_A3B,++ LLM_TYPE_31B_A3_5B,+ LLM_TYPE_80B_A3B, // Qwen3 Next+ LLM_TYPE_100B_A6B,+ LLM_TYPE_106B_A12B, // GLM-4.5-Air+diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp+index 541434888..eb135e63f 100644+--- a/src/models/nemotron-h.cpp++++ b/src/models/nemotron-h.cpp+@@ -107,12 +107,41 @@ ggml_tensor * llm_build_nemotron_h::build_attention_layer(ggml_tensor *+ }++ ggml_tensor * llm_build_nemotron_h::build_ffn_layer(ggml_tensor * cur, const llama_model & model, const int il) {+- cur = build_ffn(cur,+- model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,+- NULL, NULL, NULL,+- model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,+- NULL, LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);+- cb(cur, "ffn_out", il);++ if (model.layers[il].ffn_gate_inp == nullptr) {++ cur = build_ffn(cur,++ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,++ NULL, NULL, NULL,++ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,++ NULL,++ LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);++ cb(cur, "ffn_out", il);++ } else {++ ggml_tensor * ffn_inp = cur;++ ggml_tensor * moe_out =++ build_moe_ffn(ffn_inp,++ model.layers[il].ffn_gate_inp,++ model.layers[il].ffn_up_exps,++ nullptr, // no gate++ model.layers[il].ffn_down_exps,++ model.layers[il].ffn_exp_probs_b,++ n_expert, n_expert_used,++ LLM_FFN_RELU_SQR, hparams.expert_weights_norm,++ true, hparams.expert_weights_scale,++ LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,++ il);++ cb(moe_out, "ffn_moe_out", il);++++ ggml_tensor * ffn_shexp = build_ffn(ffn_inp,++ model.layers[il].ffn_up_shexp, NULL, NULL,++ NULL /* no gate */ , NULL, NULL,++ model.layers[il].ffn_down_shexp, NULL, NULL,++ NULL,++ LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);++ cb(ffn_shexp, "ffn_shexp", il);++++ cur = ggml_add(ctx0, moe_out, ffn_shexp);++ cb(cur, "ffn_out", il);++ }++ cur = build_cvec(cur, il);+ cb(cur, "l_out", il);diff --git a/model/parsers/nemotron3nano.go b/model/parsers/nemotron3nano.gonew file mode 100644index 000000000..c230b4e7d--- /dev/null+++ b/model/parsers/nemotron3nano.go@@ -0,0 +1,255 @@+package parsers++import (+ "regexp"+ "strings"+ "unicode"++ "github.com/ollama/ollama/api"+)++type Nemotron3NanoParserState int++const (+ Nemotron3NanoCollectingThinking Nemotron3NanoParserState = iota+ Nemotron3NanoSkipWhitespaceAfterThinking+ Nemotron3NanoCollectingContent+ Nemotron3NanoCollectingToolCalls+)++const (+ nemotronThinkClose = "</think>"+ nemotronToolCallOpen = "<tool_call>"+ nemotronToolCallClose = "</tool_call>"+)++type Nemotron3NanoParser struct {+ state Nemotron3NanoParserState+ buffer strings.Builder+ tools []api.Tool+ HasThinking bool+}++func (p *Nemotron3NanoParser) HasToolSupport() bool { return true }+func (p *Nemotron3NanoParser) HasThinkingSupport() bool { return p.HasThinking }++func (p *Nemotron3NanoParser) Init(tools []api.Tool, lastMessage *api.Message, thinkValue *api.ThinkValue) []api.Tool {+ p.tools = tools++ // Check both model capability AND request preference+ thinkingEnabled := thinkValue != nil && thinkValue.Bool()++ prefill := lastMessage != nil && lastMessage.Role == "assistant"++ if !thinkingEnabled {+ p.state = Nemotron3NanoCollectingContent+ return tools+ }++ if prefill && lastMessage.Content != "" {+ p.state = Nemotron3NanoCollectingContent+ return tools+ }++ p.state = Nemotron3NanoCollectingThinking+ return tools+}++type nemotronEvent interface {+ isNemotronEvent()+}++type nemotronEventThinkingContent struct {+ content string+}++type nemotronEventContent struct {+ content string+}++type nemotronEventToolCall struct {+ toolCall api.ToolCall+}++func (nemotronEventThinkingContent) isNemotronEvent() {}+func (nemotronEventContent) isNemotronEvent() {}+func (nemotronEventToolCall) isNemotronEvent() {}++func (p *Nemotron3NanoParser) Add(s string, done bool) (content string, thinking string, calls []api.ToolCall, err error) {+ p.buffer.WriteString(s)+ events := p.parseEvents()++ var toolCalls []api.ToolCall+ var contentSb strings.Builder+ var thinkingSb strings.Builder+ for _, event := range events {+ switch event := event.(type) {+ case nemotronEventToolCall:+ toolCalls = append(toolCalls, event.toolCall)+ case nemotronEventThinkingContent:+ thinkingSb.WriteString(event.content)+ case nemotronEventContent:+ contentSb.WriteString(event.content)+ }+ }++ return contentSb.String(), thinkingSb.String(), toolCalls, nil+}++func (p *Nemotron3NanoParser) parseEvents() []nemotronEvent {+ var all []nemotronEvent++ keepLooping := true+ for keepLooping {+ var events []nemotronEvent+ events, keepLooping = p.eat()+ if len(events) > 0 {+ all = append(all, events...)+ }+ }++ return all+}++// emitWithPartialCheck extracts unambiguous content before a potential partial tag+func (p *Nemotron3NanoParser) emitWithPartialCheck(bufStr, tag string) (unambiguous, ambiguous string) {+ if overlapLen := overlap(bufStr, tag); overlapLen > 0 {+ beforePartialTag := bufStr[:len(bufStr)-overlapLen]+ trailingLen := trailingWhitespaceLen(beforePartialTag)+ return bufStr[:len(beforePartialTag)-trailingLen], bufStr[len(beforePartialTag)-trailingLen:]+ }+ wsLen := trailingWhitespaceLen(bufStr)+ return bufStr[:len(bufStr)-wsLen], bufStr[len(bufStr)-wsLen:]+}++func (p *Nemotron3NanoParser) eat() ([]nemotronEvent, bool) {+ bufStr := p.buffer.String()+ if bufStr == "" {+ return nil, false+ }++ switch p.state {+ case Nemotron3NanoCollectingThinking:+ if strings.Contains(bufStr, nemotronThinkClose) {+ split := strings.SplitN(bufStr, nemotronThinkClose, 2)+ thinking := strings.TrimRightFunc(split[0], unicode.IsSpace)+ p.buffer.Reset()+ remainder := strings.TrimLeftFunc(split[1], unicode.IsSpace)+ p.buffer.WriteString(remainder)+ // Transition to whitespace-skipping state if buffer is empty,+ // otherwise go directly to content collection+ if remainder == "" {+ p.state = Nemotron3NanoSkipWhitespaceAfterThinking+ } else {+ p.state = Nemotron3NanoCollectingContent+ }+ if thinking != "" {+ return []nemotronEvent{nemotronEventThinkingContent{content: thinking}}, true+ }+ return nil, true+ }+ unambig, ambig := p.emitWithPartialCheck(bufStr, nemotronThinkClose)+ p.buffer.Reset()+ p.buffer.WriteString(ambig)+ if unambig != "" {+ return []nemotronEvent{nemotronEventThinkingContent{content: unambig}}, false+ }+ return nil, false++ // We only want to skip whitespace between thinking and content+ case Nemotron3NanoSkipWhitespaceAfterThinking:+ bufStr = strings.TrimLeftFunc(bufStr, unicode.IsSpace)+ p.buffer.Reset()+ p.buffer.WriteString(bufStr)+ if bufStr == "" {+ return nil, false+ }+ p.state = Nemotron3NanoCollectingContent+ return nil, true++ case Nemotron3NanoCollectingContent:+ if strings.Contains(bufStr, nemotronToolCallOpen) {+ split := strings.SplitN(bufStr, nemotronToolCallOpen, 2)+ content := strings.TrimRightFunc(split[0], unicode.IsSpace)+ p.buffer.Reset()+ p.buffer.WriteString(split[1])+ p.state = Nemotron3NanoCollectingToolCalls+ if content != "" {+ return []nemotronEvent{nemotronEventContent{content: content}}, true+ }+ return nil, true+ }+ unambig, ambig := p.emitWithPartialCheck(bufStr, nemotronToolCallOpen)+ p.buffer.Reset()+ p.buffer.WriteString(ambig)+ if unambig != "" {+ return []nemotronEvent{nemotronEventContent{content: unambig}}, false+ }+ return nil, false++ case Nemotron3NanoCollectingToolCalls:+ if strings.Contains(bufStr, nemotronToolCallClose) {+ split := strings.SplitN(bufStr, nemotronToolCallClose, 2)+ remaining := strings.TrimLeftFunc(split[1], unicode.IsSpace)+ p.buffer.Reset()+ p.buffer.WriteString(remaining)++ var events []nemotronEvent+ if tc, err := p.parseToolCall(split[0]); err == nil {+ events = append(events, nemotronEventToolCall{toolCall: tc})+ }++ if !strings.Contains(remaining, nemotronToolCallOpen) {+ p.state = Nemotron3NanoCollectingContent+ }+ return events, true+ }+ return nil, false+ }++ return nil, false+}++var (+ nemotronFunctionRegex = regexp.MustCompile(`<function=([^>]+)>`)+ nemotronParameterRegex = regexp.MustCompile(`<parameter=([^>]+)>\n?([\s\S]*?)\n?</parameter>`)+)++func (p *Nemotron3NanoParser) parseToolCall(content string) (api.ToolCall, error) {+ toolCall := api.ToolCall{}++ // Extract function name+ fnMatch := nemotronFunctionRegex.FindStringSubmatch(content)+ if len(fnMatch) < 2 {+ return toolCall, nil+ }+ toolCall.Function.Name = fnMatch[1]++ // Extract parameters+ toolCall.Function.Arguments = make(api.ToolCallFunctionArguments)+ paramMatches := nemotronParameterRegex.FindAllStringSubmatch(content, -1)+ for _, match := range paramMatches {+ if len(match) >= 3 {+ paramName := match[1]+ paramValue := strings.TrimSpace(match[2])++ // Try to parse as typed value based on tool definition+ toolCall.Function.Arguments[paramName] = p.parseParamValue(paramName, paramValue)+ }+ }++ return toolCall, nil+}++func (p *Nemotron3NanoParser) parseParamValue(paramName string, raw string) any {+ // Find the matching tool to get parameter type+ var paramType api.PropertyType+ for _, tool := range p.tools {+ if prop, ok := tool.Function.Parameters.Properties[paramName]; ok {+ paramType = prop.Type+ break+ }+ }++ return parseValue(raw, paramType)+}diff --git a/model/parsers/nemotron3nano_test.go b/model/parsers/nemotron3nano_test.gonew file mode 100644index 000000000..cfe74a314--- /dev/null+++ b/model/parsers/nemotron3nano_test.go@@ -0,0 +1,583 @@+package parsers++import (+ "testing"++ "github.com/google/go-cmp/cmp"++ "github.com/ollama/ollama/api"+)++func TestNemotron3NanoParser(t *testing.T) {+ tests := []struct {+ name string+ input string+ thinkValue *api.ThinkValue+ expectedContent string+ expectedThinking string+ expectedCalls []api.ToolCall+ }{+ {+ name: "simple content - no thinking",+ input: "Hello, how can I help you?",+ thinkValue: nil,+ expectedContent: "Hello, how can I help you?",+ },+ {+ name: "simple content - thinking disabled",+ input: "Hello, how can I help you?",+ thinkValue: &api.ThinkValue{Value: false},+ expectedContent: "Hello, how can I help you?",+ },+ {+ name: "thinking then content",+ input: "Let me think about this...</think>\nHere is my answer.",+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "Let me think about this...",+ expectedContent: "Here is my answer.",+ },+ {+ name: "thinking with newlines",+ input: "Step 1: Analyze\nStep 2: Process\nStep 3: Conclude</think>\nThe answer is 42.",+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "Step 1: Analyze\nStep 2: Process\nStep 3: Conclude",+ expectedContent: "The answer is 42.",+ },+ {+ name: "simple tool call",+ input: "<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>",+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "Paris"},+ },+ },+ },+ },+ {+ name: "content then tool call",+ input: "Let me check the weather.\n<tool_call>\n<function=get_weather>\n<parameter=city>\nNYC\n</parameter>\n</function>\n</tool_call>",+ thinkValue: nil,+ expectedContent: "Let me check the weather.",+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "NYC"},+ },+ },+ },+ },+ {+ name: "tool call with multiple parameters",+ input: "<tool_call>\n<function=book_flight>\n<parameter=from>\nSFO\n</parameter>\n<parameter=to>\nNYC\n</parameter>\n</function>\n</tool_call>",+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "book_flight",+ Arguments: map[string]any{+ "from": "SFO",+ "to": "NYC",+ },+ },+ },+ },+ },+ {+ name: "multiple tool calls",+ input: "<tool_call>\n<function=get_weather>\n<parameter=city>\nSan Francisco\n</parameter>\n</function>\n</tool_call>\n" ++ "<tool_call>\n<function=get_weather>\n<parameter=city>\nNew York\n</parameter>\n</function>\n</tool_call>",+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "San Francisco"},+ },+ },+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "New York"},+ },+ },+ },+ },+ {+ name: "thinking then tool call",+ input: "I should check the weather...</think>\n<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>",+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "I should check the weather...",+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "Paris"},+ },+ },+ },+ },+ {+ name: "thinking content then tool call",+ input: "Let me think...</think>\nI'll check for you.\n<tool_call>\n<function=search>\n<parameter=query>\ntest\n</parameter>\n</function>\n</tool_call>",+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "Let me think...",+ expectedContent: "I'll check for you.",+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "search",+ Arguments: map[string]any{"query": "test"},+ },+ },+ },+ },+ {+ name: "tool call with multiline parameter value",+ input: "<tool_call>\n<function=create_note>\n<parameter=content>\nLine 1\nLine 2\nLine 3\n</parameter>\n</function>\n</tool_call>",+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "create_note",+ Arguments: map[string]any{"content": "Line 1\nLine 2\nLine 3"},+ },+ },+ },+ },+ {+ name: "empty thinking block - immediate close",+ input: "</think>\nHere is my answer.",+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "",+ expectedContent: "Here is my answer.",+ },+ {+ name: "thinking disabled but model outputs think close anyway",+ input: "</think>\nSome content after spurious tag.",+ thinkValue: &api.ThinkValue{Value: false},+ expectedContent: "</think>\nSome content after spurious tag.",+ },+ {+ name: "tool call with no function name - returns empty tool call",+ input: "<tool_call>\n<function=>\n</function>\n</tool_call>",+ thinkValue: nil,+ expectedCalls: []api.ToolCall{{Function: api.ToolCallFunction{Name: "", Arguments: nil}}},+ },+ {+ name: "content with newlines preserved",+ input: "Line 1\n\nLine 2\n\n\nLine 3",+ thinkValue: nil,+ expectedContent: "Line 1\n\nLine 2\n\n\nLine 3",+ },+ {+ name: "thinking with only whitespace after close tag",+ input: "My thoughts...</think> \n\t\n Content here.",+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "My thoughts...",+ expectedContent: "Content here.",+ },+ {+ name: "unicode content",+ input: "Hello 世界! 🌍 Ñoño",+ thinkValue: nil,+ expectedContent: "Hello 世界! 🌍 Ñoño",+ },+ {+ name: "tool call with numeric parameter",+ input: "<tool_call>\n<function=set_temp>\n<parameter=value>\n42\n</parameter>\n</function>\n</tool_call>",+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "set_temp",+ Arguments: map[string]any{"value": "42"},+ },+ },+ },+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ p := &Nemotron3NanoParser{HasThinking: tt.thinkValue != nil && tt.thinkValue.Bool()}+ p.Init(nil, nil, tt.thinkValue)++ content, thinking, calls, err := p.Add(tt.input, false)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }++ // Drain remaining content+ finalContent, finalThinking, finalCalls, err := p.Add("", true)+ if err != nil {+ t.Fatalf("unexpected error on done: %v", err)+ }+ content += finalContent+ thinking += finalThinking+ calls = append(calls, finalCalls...)++ if diff := cmp.Diff(content, tt.expectedContent); diff != "" {+ t.Errorf("content mismatch (-got +want):\n%s", diff)+ }+ if diff := cmp.Diff(thinking, tt.expectedThinking); diff != "" {+ t.Errorf("thinking mismatch (-got +want):\n%s", diff)+ }+ if diff := cmp.Diff(calls, tt.expectedCalls); diff != "" {+ t.Errorf("calls mismatch (-got +want):\n%s", diff)+ }+ })+ }+}++func TestNemotron3NanoParser_Streaming(t *testing.T) {+ tests := []struct {+ name string+ chunks []string+ thinkValue *api.ThinkValue+ expectedContent string+ expectedThinking string+ expectedCalls []api.ToolCall+ }{+ {+ name: "streaming content character by character",+ chunks: []string{"H", "e", "l", "l", "o", ",", " ", "w", "o", "r", "l", "d", "!"},+ thinkValue: nil,+ expectedContent: "Hello, world!",+ },+ {+ name: "streaming content small tokens",+ chunks: []string{"Hel", "lo", ", ", "how ", "can", " I", " help", " you", " today", "?"},+ thinkValue: nil,+ expectedContent: "Hello, how can I help you today?",+ },+ {+ name: "streaming thinking then content - granular",+ chunks: []string{"Let", " me", " th", "ink", " about", " this", "...", "<", "/", "think", ">", "\n", "Here", " is", " my", " answer", "."},+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "Let me think about this...",+ expectedContent: "Here is my answer.",+ },+ {+ name: "streaming thinking with newlines - granular",+ chunks: []string{"Step", " 1", ":", " Ana", "lyze\n", "Step", " 2", ":", " Pro", "cess", "</", "thi", "nk>", "\n", "The", " ans", "wer."},+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "Step 1: Analyze\nStep 2: Process",+ expectedContent: "The answer.",+ },+ {+ name: "streaming tool call - highly granular",+ chunks: []string{"<", "tool", "_", "call", ">", "\n", "<", "func", "tion", "=", "get", "_", "weather", ">", "\n", "<", "param", "eter", "=", "city", ">", "\n", "Par", "is", "\n", "</", "param", "eter", ">", "\n", "</", "func", "tion", ">", "\n", "</", "tool", "_", "call", ">"},+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "Paris"},+ },+ },+ },+ },+ {+ name: "streaming content then tool call - granular",+ chunks: []string{"Let", " me", " check", " the", " weather", ".", "\n<", "tool_call", ">", "\n", "<function=", "get_weather", ">", "\n", "<parameter=", "city", ">", "\n", "NYC", "\n", "</parameter>", "\n", "</function>", "\n", "</tool_call>"},+ thinkValue: nil,+ expectedContent: "Let me check the weather.",+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "NYC"},+ },+ },+ },+ },+ {+ name: "tool call tag split character by character",+ chunks: []string{"<", "t", "o", "o", "l", "_", "c", "a", "l", "l", ">", "\n", "<", "f", "u", "n", "c", "t", "i", "o", "n", "=", "t", "e", "s", "t", ">", "\n", "<", "/", "f", "u", "n", "c", "t", "i", "o", "n", ">", "\n", "<", "/", "t", "o", "o", "l", "_", "c", "a", "l", "l", ">"},+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "test",+ Arguments: map[string]any{},+ },+ },+ },+ },+ {+ name: "thinking close tag split character by character",+ chunks: []string{"I", "'", "m", " ", "t", "h", "i", "n", "k", "i", "n", "g", ".", ".", ".", "<", "/", "t", "h", "i", "n", "k", ">", "\n", "D", "o", "n", "e", "!"},+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "I'm thinking...",+ expectedContent: "Done!",+ },+ {+ name: "multiple whitespace after think tag - separate chunks",+ chunks: []string{"Thinking...", "</think>", "\n", "\n", " ", "Content here."},+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "Thinking...",+ expectedContent: "Content here.",+ },+ {+ name: "tool call with multiple parameters - streaming",+ chunks: []string{"<tool_", "call>\n", "<function", "=book_", "flight>", "\n<para", "meter=", "from>\n", "SFO\n", "</param", "eter>", "\n<param", "eter=to", ">\nNYC", "\n</para", "meter>", "\n</func", "tion>\n", "</tool_", "call>"},+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "book_flight",+ Arguments: map[string]any{+ "from": "SFO",+ "to": "NYC",+ },+ },+ },+ },+ },+ {+ name: "thinking then content then tool call - streaming",+ chunks: []string{"Ana", "lyzing", " your", " request", "...", "</", "think", ">\n", "I'll", " check", " that", " for", " you", ".", "\n", "<tool", "_call", ">\n", "<function", "=search", ">\n", "<parameter", "=query", ">\n", "test", " query", "\n</", "parameter", ">\n", "</function", ">\n", "</tool", "_call", ">"},+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "Analyzing your request...",+ expectedContent: "I'll check that for you.",+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "search",+ Arguments: map[string]any{"query": "test query"},+ },+ },+ },+ },+ {+ name: "multiple tool calls - streaming",+ chunks: []string{+ "<tool_call>", "\n", "<function=", "get_weather>", "\n",+ "<parameter=", "city>\n", "San Fran", "cisco\n", "</parameter>", "\n",+ "</function>", "\n", "</tool_call>", "\n",+ "<tool_", "call>\n", "<function", "=get_weather", ">\n",+ "<param", "eter=city", ">\nNew", " York\n", "</parameter>\n",+ "</function>\n", "</tool_call>",+ },+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "San Francisco"},+ },+ },+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "New York"},+ },+ },+ },+ },+ {+ name: "tool call with multiline parameter - streaming",+ chunks: []string{"<tool_call>\n", "<function=", "create_note>\n", "<parameter=", "content>\n", "Line 1", "\nLine", " 2\n", "Line 3", "\n</parameter>\n", "</function>\n", "</tool_call>"},+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "create_note",+ Arguments: map[string]any{"content": "Line 1\nLine 2\nLine 3"},+ },+ },+ },+ },+ {+ name: "empty thinking block",+ chunks: []string{"</think>", "\n", "Just content."},+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "",+ expectedContent: "Just content.",+ },+ {+ name: "empty input chunks interspersed",+ chunks: []string{"Hello", "", " ", "", "world", "", "!"},+ thinkValue: nil,+ expectedContent: "Hello world!",+ },+ {+ name: "tool call immediately after think close - no content",+ chunks: []string{"Analyzing...", "</think>", "\n", "<tool_call>", "\n<function=test>\n</function>\n", "</tool_call>"},+ thinkValue: &api.ThinkValue{Value: true},+ expectedThinking: "Analyzing...",+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "test",+ Arguments: map[string]any{},+ },+ },+ },+ },+ {+ name: "tool call with empty parameter value",+ chunks: []string{"<tool_call>\n<function=test>\n<parameter=name>\n", "\n</parameter>\n</function>\n</tool_call>"},+ thinkValue: nil,+ expectedCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "test",+ Arguments: map[string]any{"name": ""},+ },+ },+ },+ },+ {+ name: "partial tool call tag at end - buffered",+ chunks: []string{"Here's some content", "<tool"},+ thinkValue: nil,+ expectedContent: "Here's some content",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ p := &Nemotron3NanoParser{HasThinking: tt.thinkValue != nil && tt.thinkValue.Bool()}+ p.Init(nil, nil, tt.thinkValue)++ var allContent string+ var allThinking string+ var allCalls []api.ToolCall++ for _, chunk := range tt.chunks {+ content, thinking, calls, err := p.Add(chunk, false)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ allContent += content+ allThinking += thinking+ allCalls = append(allCalls, calls...)+ }++ // Drain+ content, thinking, calls, err := p.Add("", true)+ if err != nil {+ t.Fatalf("unexpected error on done: %v", err)+ }+ allContent += content+ allThinking += thinking+ allCalls = append(allCalls, calls...)++ if diff := cmp.Diff(allContent, tt.expectedContent); diff != "" {+ t.Errorf("content mismatch (-got +want):\n%s", diff)+ }+ if diff := cmp.Diff(allThinking, tt.expectedThinking); diff != "" {+ t.Errorf("thinking mismatch (-got +want):\n%s", diff)+ }+ if diff := cmp.Diff(allCalls, tt.expectedCalls); diff != "" {+ t.Errorf("calls mismatch (-got +want):\n%s", diff)+ }+ })+ }+}++func TestNemotron3NanoParser_HasToolSupport(t *testing.T) {+ p := &Nemotron3NanoParser{}+ if !p.HasToolSupport() {+ t.Error("expected HasToolSupport to return true")+ }+}++func TestNemotron3NanoParser_HasThinkingSupport(t *testing.T) {+ t.Run("with thinking enabled", func(t *testing.T) {+ p := &Nemotron3NanoParser{HasThinking: true}+ if !p.HasThinkingSupport() {+ t.Error("expected HasThinkingSupport to return true")+ }+ })++ t.Run("with thinking disabled", func(t *testing.T) {+ p := &Nemotron3NanoParser{HasThinking: false}+ if p.HasThinkingSupport() {+ t.Error("expected HasThinkingSupport to return false")+ }+ })+}++func TestNemotron3NanoParser_Init(t *testing.T) {+ t.Run("starts in thinking state when enabled", func(t *testing.T) {+ p := &Nemotron3NanoParser{HasThinking: true}+ p.Init(nil, nil, &api.ThinkValue{Value: true})+ if p.state != Nemotron3NanoCollectingThinking {+ t.Errorf("expected state Nemotron3NanoCollectingThinking, got %v", p.state)+ }+ })++ t.Run("starts in content state when thinking disabled", func(t *testing.T) {+ p := &Nemotron3NanoParser{HasThinking: true}+ p.Init(nil, nil, &api.ThinkValue{Value: false})+ if p.state != Nemotron3NanoCollectingContent {+ t.Errorf("expected state Nemotron3NanoCollectingContent, got %v", p.state)+ }+ })++ t.Run("starts in content state when nil thinkValue", func(t *testing.T) {+ p := &Nemotron3NanoParser{HasThinking: true}+ p.Init(nil, nil, nil)+ if p.state != Nemotron3NanoCollectingContent {+ t.Errorf("expected state Nemotron3NanoCollectingContent, got %v", p.state)+ }+ })++ t.Run("starts in content state with assistant prefill", func(t *testing.T) {+ p := &Nemotron3NanoParser{HasThinking: true}+ prefill := &api.Message{Role: "assistant", Content: "Starting..."}+ p.Init(nil, prefill, &api.ThinkValue{Value: true})+ if p.state != Nemotron3NanoCollectingContent {+ t.Errorf("expected state Nemotron3NanoCollectingContent, got %v", p.state)+ }+ })+}++func TestNemotron3NanoParser_WithTools(t *testing.T) {+ tools := []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_weather",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Properties: map[string]api.ToolProperty{+ "city": {Type: api.PropertyType{"string"}},+ },+ },+ },+ },+ }++ p := &Nemotron3NanoParser{}+ returnedTools := p.Init(tools, nil, nil)++ if diff := cmp.Diff(returnedTools, tools); diff != "" {+ t.Errorf("tools mismatch (-got +want):\n%s", diff)+ }++ // Parse a tool call+ input := "<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>"+ _, _, calls, err := p.Add(input, true)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }++ expectedCalls := []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "Paris"},+ },+ },+ }++ if diff := cmp.Diff(calls, expectedCalls); diff != "" {+ t.Errorf("calls mismatch (-got +want):\n%s", diff)+ }+}diff --git a/model/parsers/parsers.go b/model/parsers/parsers.goindex ab52267cb..417261b13 100644--- a/model/parsers/parsers.go+++ b/model/parsers/parsers.go@@ -62,6 +62,10 @@ func ParserForName(name string) Parser {return &Olmo3Parser{}case "olmo3-think":return &Olmo3ThinkParser{}+ case "nemotron-3-nano":+ return &Nemotron3NanoParser{HasThinking: false}+ case "nemotron-3-nano-thinking":+ return &Nemotron3NanoParser{HasThinking: true}default:return nil}diff --git a/model/renderers/nemotron3nano.go b/model/renderers/nemotron3nano.gonew file mode 100644index 000000000..08074d18c--- /dev/null+++ b/model/renderers/nemotron3nano.go@@ -0,0 +1,222 @@+package renderers++import (+ "encoding/json"+ "fmt"+ "strings"++ "github.com/ollama/ollama/api"+)++type Nemotron3NanoRenderer struct {+ IsThinking bool+}++func (r *Nemotron3NanoRenderer) Render(messages []api.Message, tools []api.Tool, thinkValue *api.ThinkValue) (string, error) {+ var sb strings.Builder++ // thinking is enabled: model must support it AND user must request it+ enableThinking := r.IsThinking && (thinkValue != nil && thinkValue.Bool())++ // Extract system message if present+ var systemMessage string+ var loopMessages []api.Message+ if len(messages) > 0 && messages[0].Role == "system" {+ systemMessage = messages[0].Content+ loopMessages = messages[1:]+ } else {+ loopMessages = messages+ }++ // Find last user message index for thinking truncation+ lastUserIdx := -1+ for i, msg := range loopMessages {+ if msg.Role == "user" {+ lastUserIdx = i+ }+ }++ sb.WriteString("<|im_start|>system\n")+ if systemMessage != "" {+ sb.WriteString(systemMessage)+ }++ if len(tools) > 0 {+ if systemMessage != "" {+ sb.WriteString("\n\n")+ }+ sb.WriteString(r.renderTools(tools))+ }+ sb.WriteString("<|im_end|>\n")++ for i, message := range loopMessages {+ switch message.Role {+ case "assistant":+ // Build content with thinking tags+ content := r.buildContent(message)+ shouldTruncate := i < lastUserIdx++ if len(message.ToolCalls) > 0 {+ sb.WriteString("<|im_start|>assistant\n")+ sb.WriteString(r.formatContent(content, shouldTruncate, true))+ r.writeToolCalls(&sb, message.ToolCalls)+ sb.WriteString("<|im_end|>\n")+ } else {+ formatted := r.formatContent(content, shouldTruncate, false)+ sb.WriteString("<|im_start|>assistant\n" + formatted + "<|im_end|>\n")+ }++ case "user", "system":+ sb.WriteString("<|im_start|>" + message.Role + "\n")+ sb.WriteString(message.Content)+ sb.WriteString("<|im_end|>\n")++ case "tool":+ // Check if previous message was also a tool message+ prevWasTool := i > 0 && loopMessages[i-1].Role == "tool"+ nextIsTool := i+1 < len(loopMessages) && loopMessages[i+1].Role == "tool"++ if !prevWasTool {+ sb.WriteString("<|im_start|>user\n")+ }+ sb.WriteString("<tool_response>\n")+ sb.WriteString(message.Content)+ sb.WriteString("\n</tool_response>\n")++ if !nextIsTool {+ sb.WriteString("<|im_end|>\n")+ }++ default:+ sb.WriteString("<|im_start|>" + message.Role + "\n" + message.Content + "<|im_end|>\n")+ }+ }++ // Add generation prompt+ if enableThinking {+ sb.WriteString("<|im_start|>assistant\n<think>\n")+ } else {+ sb.WriteString("<|im_start|>assistant\n<think></think>")+ }++ return sb.String(), nil+}++func (r *Nemotron3NanoRenderer) renderTools(tools []api.Tool) string {+ var sb strings.Builder+ sb.WriteString("# Tools\n\nYou have access to the following functions:\n\n<tools>")++ for _, tool := range tools {+ fn := tool.Function+ sb.WriteString("\n<function>\n<name>" + fn.Name + "</name>")++ if fn.Description != "" {+ sb.WriteString("\n<description>" + strings.TrimSpace(fn.Description) + "</description>")+ }++ sb.WriteString("\n<parameters>")+ if fn.Parameters.Properties != nil {+ for paramName, paramFields := range fn.Parameters.Properties {+ sb.WriteString("\n<parameter>")+ sb.WriteString("\n<name>" + paramName + "</name>")++ if len(paramFields.Type) > 0 {+ sb.WriteString("\n<type>" + strings.Join(paramFields.Type, ", ") + "</type>")+ }++ if paramFields.Description != "" {+ sb.WriteString("\n<description>" + strings.TrimSpace(paramFields.Description) + "</description>")+ }++ if len(paramFields.Enum) > 0 {+ enumJSON, _ := json.Marshal(paramFields.Enum)+ sb.WriteString("\n<enum>" + string(enumJSON) + "</enum>")+ }++ sb.WriteString("\n</parameter>")+ }+ }++ if len(fn.Parameters.Required) > 0 {+ reqJSON, _ := json.Marshal(fn.Parameters.Required)+ sb.WriteString("\n<required>" + string(reqJSON) + "</required>")+ }++ sb.WriteString("\n</parameters>")+ sb.WriteString("\n</function>")+ }++ sb.WriteString("\n</tools>")++ sb.WriteString("\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>")++ return sb.String()+}++func (r *Nemotron3NanoRenderer) buildContent(message api.Message) string {+ // The parser always extracts thinking into the Thinking field,+ // so Content will never have <think> tags embedded+ if message.Thinking != "" {+ return "<think>\n" + message.Thinking + "\n</think>\n" + message.Content+ }+ return "<think></think>" + message.Content+}++func (r *Nemotron3NanoRenderer) formatContent(content string, truncate bool, addNewline bool) string {+ if content == "" {+ return "<think></think>"+ }++ if !truncate {+ if addNewline {+ return strings.TrimSpace(content) + "\n"+ }+ return strings.TrimSpace(content)+ }++ // Truncate thinking - keep only content after </think>+ c := content+ if strings.Contains(c, "</think>") {+ parts := strings.Split(c, "</think>")+ c = parts[len(parts)-1]+ } else if strings.Contains(c, "<think>") {+ parts := strings.Split(c, "<think>")+ c = parts[0]+ }+ c = "<think></think>" + strings.TrimSpace(c)++ if addNewline && len(c) > len("<think></think>") {+ return c + "\n"+ }+ if c == "<think></think>" {+ return c+ }+ return strings.TrimSpace(c)+}++func (r *Nemotron3NanoRenderer) writeToolCalls(sb *strings.Builder, toolCalls []api.ToolCall) {+ for _, tc := range toolCalls {+ sb.WriteString("<tool_call>\n<function=" + tc.Function.Name + ">\n")+ for name, value := range tc.Function.Arguments {+ sb.WriteString("<parameter=" + name + ">\n" + r.formatArgValue(value) + "\n</parameter>\n")+ }+ sb.WriteString("</function>\n</tool_call>\n")+ }+}++func (r *Nemotron3NanoRenderer) formatArgValue(value any) string {+ switch v := value.(type) {+ case map[string]any, []any:+ jsonBytes, _ := json.Marshal(v)+ return string(jsonBytes)+ default:+ return fmt.Sprintf("%v", v)+ }+}diff --git a/model/renderers/nemotron3nano_test.go b/model/renderers/nemotron3nano_test.gonew file mode 100644index 000000000..dad528cc9--- /dev/null+++ b/model/renderers/nemotron3nano_test.go@@ -0,0 +1,585 @@+package renderers++import (+ "testing"++ "github.com/google/go-cmp/cmp"++ "github.com/ollama/ollama/api"+)++func TestNemotron3NanoRenderer(t *testing.T) {+ tests := []struct {+ name string+ msgs []api.Message+ tools []api.Tool+ thinkValue *api.ThinkValue+ isThinking bool+ expected string+ }{+ {+ name: "basic user message - thinking mode",+ msgs: []api.Message{+ {Role: "user", Content: "Hello!"},+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n<|im_end|>\n" ++ "<|im_start|>user\nHello!<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "basic user message - no thinking",+ msgs: []api.Message{+ {Role: "user", Content: "Hello!"},+ },+ isThinking: false,+ expected: "<|im_start|>system\n<|im_end|>\n" ++ "<|im_start|>user\nHello!<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>",+ },+ {+ name: "with system message",+ msgs: []api.Message{+ {Role: "system", Content: "You are a helpful assistant."},+ {Role: "user", Content: "Hello!"},+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" ++ "<|im_start|>user\nHello!<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "multi-turn conversation",+ msgs: []api.Message{+ {Role: "user", Content: "Hi"},+ {Role: "assistant", Content: "Hello! How can I help?"},+ {Role: "user", Content: "Tell me a joke"},+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n<|im_end|>\n" ++ "<|im_start|>user\nHi<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>Hello! How can I help?<|im_end|>\n" ++ "<|im_start|>user\nTell me a joke<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "with tools",+ msgs: []api.Message{+ {Role: "user", Content: "What's the weather in Paris?"},+ },+ tools: []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_weather",+ Description: "Get the current weather",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Required: []string{"city"},+ Properties: map[string]api.ToolProperty{+ "city": {Type: api.PropertyType{"string"}, Description: "The city name"},+ },+ },+ },+ },+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n" ++ "# Tools\n\nYou have access to the following functions:\n\n<tools>\n" ++ "<function>\n<name>get_weather</name>\n" ++ "<description>Get the current weather</description>\n" ++ "<parameters>\n" ++ "<parameter>\n<name>city</name>\n<type>string</type>\n<description>The city name</description>\n</parameter>\n" ++ "<required>[\"city\"]</required>\n" ++ "</parameters>\n</function>\n</tools>\n\n" ++ "If you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" ++ "</IMPORTANT><|im_end|>\n" ++ "<|im_start|>user\nWhat's the weather in Paris?<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "tool call with response",+ msgs: []api.Message{+ {Role: "user", Content: "What's the weather in Paris?"},+ {+ Role: "assistant",+ ToolCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "Paris"},+ },+ },+ },+ },+ {Role: "tool", Content: "Sunny, 72F"},+ },+ tools: []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_weather",+ Description: "Get the current weather",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Required: []string{"city"},+ Properties: map[string]api.ToolProperty{+ "city": {Type: api.PropertyType{"string"}, Description: "The city name"},+ },+ },+ },+ },+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n" ++ "# Tools\n\nYou have access to the following functions:\n\n<tools>\n" ++ "<function>\n<name>get_weather</name>\n" ++ "<description>Get the current weather</description>\n" ++ "<parameters>\n" ++ "<parameter>\n<name>city</name>\n<type>string</type>\n<description>The city name</description>\n</parameter>\n" ++ "<required>[\"city\"]</required>\n" ++ "</parameters>\n</function>\n</tools>\n\n" ++ "If you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" ++ "</IMPORTANT><|im_end|>\n" ++ "<|im_start|>user\nWhat's the weather in Paris?<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>\n" ++ "<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>\n<|im_end|>\n" ++ "<|im_start|>user\n<tool_response>\nSunny, 72F\n</tool_response>\n<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "assistant with content and tool call",+ msgs: []api.Message{+ {Role: "user", Content: "What's the weather?"},+ {+ Role: "assistant",+ Content: "Let me check that for you.",+ ToolCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "Paris"},+ },+ },+ },+ },+ {Role: "tool", Content: "Sunny"},+ },+ tools: []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_weather",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Properties: map[string]api.ToolProperty{+ "city": {Type: api.PropertyType{"string"}},+ },+ },+ },+ },+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n" ++ "# Tools\n\nYou have access to the following functions:\n\n<tools>\n" ++ "<function>\n<name>get_weather</name>\n" ++ "<parameters>\n" ++ "<parameter>\n<name>city</name>\n<type>string</type>\n</parameter>\n" ++ "</parameters>\n</function>\n</tools>\n\n" ++ "If you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" ++ "</IMPORTANT><|im_end|>\n" ++ "<|im_start|>user\nWhat's the weather?<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>Let me check that for you.\n" ++ "<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>\n<|im_end|>\n" ++ "<|im_start|>user\n<tool_response>\nSunny\n</tool_response>\n<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "thinking in history is truncated",+ msgs: []api.Message{+ {Role: "user", Content: "Hi"},+ {Role: "assistant", Content: "Hello!", Thinking: "Let me think about this..."},+ {Role: "user", Content: "How are you?"},+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n<|im_end|>\n" ++ "<|im_start|>user\nHi<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>Hello!<|im_end|>\n" ++ "<|im_start|>user\nHow are you?<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "parallel tool calls",+ msgs: []api.Message{+ {Role: "user", Content: "Weather in Paris and London?"},+ {+ Role: "assistant",+ ToolCalls: []api.ToolCall{+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "Paris"},+ },+ },+ {+ Function: api.ToolCallFunction{+ Name: "get_weather",+ Arguments: map[string]any{"city": "London"},+ },+ },+ },+ },+ {Role: "tool", Content: "Sunny"},+ {Role: "tool", Content: "Rainy"},+ },+ tools: []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_weather",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Properties: map[string]api.ToolProperty{+ "city": {Type: api.PropertyType{"string"}},+ },+ },+ },+ },+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n" ++ "# Tools\n\nYou have access to the following functions:\n\n<tools>\n" ++ "<function>\n<name>get_weather</name>\n" ++ "<parameters>\n" ++ "<parameter>\n<name>city</name>\n<type>string</type>\n</parameter>\n" ++ "</parameters>\n</function>\n</tools>\n\n" ++ "If you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" ++ "</IMPORTANT><|im_end|>\n" ++ "<|im_start|>user\nWeather in Paris and London?<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>\n" ++ "<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>\n" ++ "<tool_call>\n<function=get_weather>\n<parameter=city>\nLondon\n</parameter>\n</function>\n</tool_call>\n<|im_end|>\n" ++ "<|im_start|>user\n<tool_response>\nSunny\n</tool_response>\n<tool_response>\nRainy\n</tool_response>\n<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "thinking disabled even when model supports it",+ msgs: []api.Message{+ {Role: "user", Content: "Hello!"},+ },+ isThinking: true, // model supports thinking+ thinkValue: nil, // but user didn't request it+ expected: "<|im_start|>system\n<|im_end|>\n" ++ "<|im_start|>user\nHello!<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>",+ },+ {+ name: "complex message history with thinking, tools, tool calls, tool results and content",+ msgs: []api.Message{+ {Role: "user", Content: "What's the weather in Paris and London? Also, what's 2+2?"},+ {Role: "assistant", Content: "", Thinking: "I need to check the weather for both cities and calculate 2+2. Let me start with the weather calls.", ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{Name: "get_weather", Arguments: api.ToolCallFunctionArguments{"city": "Paris"}}},+ {Function: api.ToolCallFunction{Name: "get_weather", Arguments: api.ToolCallFunctionArguments{"city": "London"}}},+ }},+ {Role: "tool", Content: "Sunny, 22°C", ToolCallID: "call1"},+ {Role: "tool", Content: "Rainy, 15°C", ToolCallID: "call2"},+ {Role: "assistant", Content: "", Thinking: "Now I have the weather data. Let me calculate 2+2.", ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{Name: "calculate", Arguments: api.ToolCallFunctionArguments{"expression": "2+2"}}},+ }},+ {Role: "tool", Content: "4", ToolCallID: "call3"},+ {Role: "assistant", Content: "Based on the weather data, Paris is sunny at 22°C and London is rainy at 15°C. Also, 2+2 equals 4.", Thinking: "Perfect! I have all the information needed to provide a complete answer."},+ },+ tools: []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_weather",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Properties: map[string]api.ToolProperty{+ "city": {Type: api.PropertyType{"string"}},+ },+ },+ },+ },+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "calculate",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Properties: map[string]api.ToolProperty{+ "expression": {Type: api.PropertyType{"string"}},+ },+ },+ },+ },+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n" ++ "# Tools\n\nYou have access to the following functions:\n\n<tools>\n" ++ "<function>\n<name>get_weather</name>\n" ++ "<parameters>\n" ++ "<parameter>\n<name>city</name>\n<type>string</type>\n</parameter>\n" ++ "</parameters>\n</function>\n" ++ "<function>\n<name>calculate</name>\n" ++ "<parameters>\n" ++ "<parameter>\n<name>expression</name>\n<type>string</type>\n</parameter>\n" ++ "</parameters>\n</function>\n</tools>\n\n" ++ "If you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" ++ "</IMPORTANT><|im_end|>\n" ++ "<|im_start|>user\nWhat's the weather in Paris and London? Also, what's 2+2?<|im_end|>\n" ++ "<|im_start|>assistant\n" ++ "<think>\nI need to check the weather for both cities and calculate 2+2. Let me start with the weather calls.\n</think>\n" ++ "<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>\n" ++ "<tool_call>\n<function=get_weather>\n<parameter=city>\nLondon\n</parameter>\n</function>\n</tool_call>\n<|im_end|>\n" ++ "<|im_start|>user\n<tool_response>\nSunny, 22°C\n</tool_response>\n<tool_response>\nRainy, 15°C\n</tool_response>\n<|im_end|>\n" ++ "<|im_start|>assistant\n" ++ "<think>\nNow I have the weather data. Let me calculate 2+2.\n</think>\n" ++ "<tool_call>\n<function=calculate>\n<parameter=expression>\n2+2\n</parameter>\n</function>\n</tool_call>\n<|im_end|>\n" ++ "<|im_start|>user\n<tool_response>\n4\n</tool_response>\n<|im_end|>\n" ++ "<|im_start|>assistant\n" ++ "<think>\nPerfect! I have all the information needed to provide a complete answer.\n</think>\n" ++ "Based on the weather data, Paris is sunny at 22°C and London is rainy at 15°C. Also, 2+2 equals 4.<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "empty messages list",+ msgs: []api.Message{},+ isThinking: false,+ expected: "<|im_start|>system\n<|im_end|>\n<|im_start|>assistant\n<think></think>",+ },+ {+ name: "tool result with JSON content",+ msgs: []api.Message{+ {Role: "user", Content: "Get user info"},+ {+ Role: "assistant",+ ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{Name: "get_user", Arguments: map[string]any{"id": "123"}}},+ },+ },+ {Role: "tool", Content: `{"name": "John", "age": 30, "active": true}`},+ },+ tools: []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "get_user",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Properties: map[string]api.ToolProperty{"id": {Type: api.PropertyType{"string"}}},+ },+ },+ },+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n" ++ "# Tools\n\nYou have access to the following functions:\n\n<tools>\n" ++ "<function>\n<name>get_user</name>\n<parameters>\n" ++ "<parameter>\n<name>id</name>\n<type>string</type>\n</parameter>\n" ++ "</parameters>\n</function>\n</tools>\n\n" ++ "If you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" ++ "</IMPORTANT><|im_end|>\n" ++ "<|im_start|>user\nGet user info<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>\n" ++ "<tool_call>\n<function=get_user>\n<parameter=id>\n123\n</parameter>\n</function>\n</tool_call>\n<|im_end|>\n" ++ "<|im_start|>user\n<tool_response>\n{\"name\": \"John\", \"age\": 30, \"active\": true}\n</tool_response>\n<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "assistant message with only thinking no content",+ msgs: []api.Message{+ {Role: "user", Content: "Think about this"},+ {Role: "assistant", Thinking: "Deep thoughts here...", Content: ""},+ {Role: "user", Content: "What did you think?"},+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n<|im_end|>\n" ++ "<|im_start|>user\nThink about this<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think><|im_end|>\n" ++ "<|im_start|>user\nWhat did you think?<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "tool call with complex nested argument",+ msgs: []api.Message{+ {Role: "user", Content: "Create data"},+ {+ Role: "assistant",+ ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{+ Name: "create",+ Arguments: map[string]any{+ "data": map[string]any{"nested": "value", "count": 42},+ },+ }},+ },+ },+ {Role: "tool", Content: "Created"},+ },+ tools: []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "create",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Properties: map[string]api.ToolProperty{"data": {Type: api.PropertyType{"object"}}},+ },+ },+ },+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n" ++ "# Tools\n\nYou have access to the following functions:\n\n<tools>\n" ++ "<function>\n<name>create</name>\n<parameters>\n" ++ "<parameter>\n<name>data</name>\n<type>object</type>\n</parameter>\n" ++ "</parameters>\n</function>\n</tools>\n\n" ++ "If you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" ++ "</IMPORTANT><|im_end|>\n" ++ "<|im_start|>user\nCreate data<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>\n" ++ "<tool_call>\n<function=create>\n<parameter=data>\n{\"count\":42,\"nested\":\"value\"}\n</parameter>\n</function>\n</tool_call>\n<|im_end|>\n" ++ "<|im_start|>user\n<tool_response>\nCreated\n</tool_response>\n<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "content explaining the format itself",+ msgs: []api.Message{+ {Role: "user", Content: "How do I format a tool call?"},+ {Role: "assistant", Content: "To call a tool, use <tool_call> tags with <function=name> inside."},+ {Role: "user", Content: "Thanks!"},+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n<|im_end|>\n" ++ "<|im_start|>user\nHow do I format a tool call?<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>To call a tool, use <tool_call> tags with <function=name> inside.<|im_end|>\n" ++ "<|im_start|>user\nThanks!<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ {+ name: "unicode in content and tool args",+ msgs: []api.Message{+ {Role: "user", Content: "Translate 你好"},+ {+ Role: "assistant",+ ToolCalls: []api.ToolCall{+ {Function: api.ToolCallFunction{Name: "translate", Arguments: map[string]any{"text": "你好"}}},+ },+ },+ {Role: "tool", Content: "Hello"},+ },+ tools: []api.Tool{+ {+ Type: "function",+ Function: api.ToolFunction{+ Name: "translate",+ Parameters: api.ToolFunctionParameters{+ Type: "object",+ Properties: map[string]api.ToolProperty{+ "text": {Type: api.PropertyType{"string"}},+ },+ },+ },+ },+ },+ isThinking: true,+ thinkValue: &api.ThinkValue{Value: true},+ expected: "<|im_start|>system\n" ++ "# Tools\n\nYou have access to the following functions:\n\n<tools>\n" ++ "<function>\n<name>translate</name>\n<parameters>\n" ++ "<parameter>\n<name>text</name>\n<type>string</type>\n</parameter>\n" ++ "</parameters>\n</function>\n</tools>\n\n" ++ "If you choose to call a function ONLY reply in the following format with NO suffix:\n\n" ++ "<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n" ++ "<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n" ++ "</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n" ++ "- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n" ++ "- Required parameters MUST be specified\n" ++ "- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n" ++ "- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n" ++ "</IMPORTANT><|im_end|>\n" ++ "<|im_start|>user\nTranslate 你好<|im_end|>\n" ++ "<|im_start|>assistant\n<think></think>\n" ++ "<tool_call>\n<function=translate>\n<parameter=text>\n你好\n</parameter>\n</function>\n</tool_call>\n<|im_end|>\n" ++ "<|im_start|>user\n<tool_response>\nHello\n</tool_response>\n<|im_end|>\n" ++ "<|im_start|>assistant\n<think>\n",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ renderer := &Nemotron3NanoRenderer{IsThinking: tt.isThinking}+ rendered, err := renderer.Render(tt.msgs, tt.tools, tt.thinkValue)+ if err != nil {+ t.Fatal(err)+ }+ if diff := cmp.Diff(rendered, tt.expected); diff != "" {+ t.Errorf("mismatch (-got +want):\n%s", diff)+ }+ })+ }+}diff --git a/model/renderers/renderer.go b/model/renderers/renderer.goindex d05182d94..7c03cf93f 100644--- a/model/renderers/renderer.go+++ b/model/renderers/renderer.go@@ -76,6 +76,12 @@ func rendererForName(name string) Renderer {// Used for Olmo-3-32B-Thinkrenderer := &Olmo3ThinkRenderer{Variant: Olmo3Think32B}return renderer+ case "nemotron-3-nano":+ renderer := &Nemotron3NanoRenderer{IsThinking: false}+ return renderer+ case "nemotron-3-nano-thinking":+ renderer := &Nemotron3NanoRenderer{IsThinking: true}+ return rendererdefault:return nil}