LLM inference in C/C++
C++ ggml
convert : fix token configuration for PLaMo-3 (#29843)
Toki Nasin7481354a1747
diff --git a/conversion/base.py b/conversion/base.pyindex 3b6fe04c2d..306a9f0f24 100644--- a/conversion/base.py+++ b/conversion/base.py@@ -2499,7 +2499,11 @@ class TextModel(ModelBase):if template is not None:self.gguf_writer.add_chat_template(template)- def _set_vocab_plamo(self):+ def _set_vocab_plamo(+ self,+ eot_token: str,+ normal_tokens: Iterable[str] = (),+ ):# PLaMo models use a custom tokenizer with a .jsonl filetokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"tokenizer_config_path = self.dir_model / "tokenizer_config.json"@@ -2523,27 +2527,30 @@ class TextModel(ModelBase):tokens = []scores = []toktypes = []+ normal_tokens = set(normal_tokens)with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:for line_num, line in enumerate(f):if line.strip():token_data = json.loads(line)# Format: [token, score, type, ?, ?, ?, ?]- token = token_data[0].encode("utf-8")+ token_str = token_data[0]+ token = token_str.encode("utf-8")score = float(token_data[1])token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"tokens.append(token)scores.append(score)- if token_type_str == "UNKNOWN":+ if token_str in normal_tokens:+ toktypes.append(gguf.TokenType.NORMAL)+ elif token_type_str == "UNKNOWN":toktypes.append(gguf.TokenType.UNKNOWN)elif token_type_str == "CONTROL":toktypes.append(gguf.TokenType.CONTROL)elif token_type_str == "BYTE":toktypes.append(gguf.TokenType.BYTE)else:- token_str = token_data[0]if token_str.startswith("<|plamo:") and token_str.endswith("|>"):toktypes.append(gguf.TokenType.CONTROL)else:@@ -2580,8 +2587,7 @@ class TextModel(ModelBase):token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))self.gguf_writer.add_unk_token_id(token_id)- # Add <|plamo:op|> as EOT to ensure appropriate end of generation- self.gguf_writer.add_eot_token_id(4)+ self.gguf_writer.add_eot_token_id(tokens.index(eot_token.encode("utf-8")))self.gguf_writer.add_add_space_prefix(False)diff --git a/conversion/plamo.py b/conversion/plamo.pyindex 52055e4a16..a843e059a1 100644--- a/conversion/plamo.py+++ b/conversion/plamo.py@@ -64,7 +64,7 @@ class Plamo2Model(TextModel):model_arch = gguf.MODEL_ARCH.PLAMO2def set_vocab(self):- self._set_vocab_plamo()+ self._set_vocab_plamo(eot_token="<|plamo:op|>")def set_gguf_parameters(self):hparams = self.hparams@@ -170,7 +170,10 @@ class Plamo3Model(TextModel):})def set_vocab(self):- self._set_vocab_plamo()+ self._set_vocab_plamo(+ eot_token="<|plamo:tag|>",+ normal_tokens=("<|plamo:begin_", "<|plamo:end_", ":plamo|>"),+ )tokenizer_config_path = self.dir_model / "tokenizer_config.json"tokenizer_config = {}