LLM inference in C/C++
C++ ggml
convert : export YaRN truncation config (#30288)
Takamichi Oharaeca5458ce034
diff --git a/conversion/base.py b/conversion/base.pyindex fb57503264..efb7ea72e2 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -1643,6 +1643,8 @@ class TextModel(ModelBase):self.gguf_writer.add_rope_scaling_type(rope_gguf_type)self.gguf_writer.add_rope_scaling_factor(rope_factor)self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_params["original_max_position_embeddings"])+ if (truncate := rope_params.get("truncate")) is not None:+ self.gguf_writer.add_rope_scaling_yarn_truncate(truncate)if (yarn_ext_factor := rope_params.get("extrapolation_factor")) is not None:self.gguf_writer.add_rope_scaling_yarn_ext_factor(yarn_ext_factor)if (yarn_attn_factor := rope_params.get("attention_factor", rope_params.get("attn_factor"))) is not None:diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 8dd2ff58cb..e5b236b112 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -268,6 +268,7 @@ class Keys:SCALING_ATTN_FACTOR = "{arch}.rope.scaling.attn_factor"SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length"SCALING_FINETUNED = "{arch}.rope.scaling.finetuned"+ SCALING_YARN_TRUNCATE = "{arch}.rope.scaling.yarn_truncate"SCALING_YARN_LOG_MUL = "{arch}.rope.scaling.yarn_log_multiplier"SCALING_YARN_EXT_FACTOR = "{arch}.rope.scaling.yarn_ext_factor"SCALING_YARN_ATTN_FACTOR = "{arch}.rope.scaling.yarn_attn_factor"diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex cda59a6285..6b12ad23a5 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -1179,6 +1179,9 @@ class GGUFWriter:def add_rope_scaling_finetuned(self, value: bool) -> None:self.add_bool(Keys.Rope.SCALING_FINETUNED.format(arch=self.arch), value)+ def add_rope_scaling_yarn_truncate(self, value: bool) -> None:+ self.add_bool(Keys.Rope.SCALING_YARN_TRUNCATE.format(arch=self.arch), value)+def add_rope_scaling_yarn_log_mul(self, value: float) -> None:self.add_float32(Keys.Rope.SCALING_YARN_LOG_MUL.format(arch=self.arch), value)