Skip to content

LLM inference in C/C++

C++ ggml

convert : fix token configuration for PLaMo-3 (#29843)

Toki Nasin7481354a1747
diff --git a/conversion/base.py b/conversion/base.py
index 3b6fe04c2d..306a9f0f24 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -2499,7 +2499,11 @@ class TextModel(ModelBase):
if template is not None:
self.gguf_writer.add_chat_template(template)
- def _set_vocab_plamo(self):
+ def _set_vocab_plamo(
+ self,
+ eot_token: str,
+ normal_tokens: Iterable[str] = (),
+ ):
# PLaMo models use a custom tokenizer with a .jsonl file
tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"
tokenizer_config_path = self.dir_model / "tokenizer_config.json"
@@ -2523,27 +2527,30 @@ class TextModel(ModelBase):
tokens = []
scores = []
toktypes = []
+ normal_tokens = set(normal_tokens)
with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:
for line_num, line in enumerate(f):
if line.strip():
token_data = json.loads(line)
# Format: [token, score, type, ?, ?, ?, ?]
- token = token_data[0].encode("utf-8")
+ token_str = token_data[0]
+ token = token_str.encode("utf-8")
score = float(token_data[1])
token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"
tokens.append(token)
scores.append(score)
- if token_type_str == "UNKNOWN":
+ if token_str in normal_tokens:
+ toktypes.append(gguf.TokenType.NORMAL)
+ elif token_type_str == "UNKNOWN":
toktypes.append(gguf.TokenType.UNKNOWN)
elif token_type_str == "CONTROL":
toktypes.append(gguf.TokenType.CONTROL)
elif token_type_str == "BYTE":
toktypes.append(gguf.TokenType.BYTE)
else:
- token_str = token_data[0]
if token_str.startswith("<|plamo:") and token_str.endswith("|>"):
toktypes.append(gguf.TokenType.CONTROL)
else:
@@ -2580,8 +2587,7 @@ class TextModel(ModelBase):
token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))
self.gguf_writer.add_unk_token_id(token_id)
- # Add <|plamo:op|> as EOT to ensure appropriate end of generation
- self.gguf_writer.add_eot_token_id(4)
+ self.gguf_writer.add_eot_token_id(tokens.index(eot_token.encode("utf-8")))
self.gguf_writer.add_add_space_prefix(False)
diff --git a/conversion/plamo.py b/conversion/plamo.py
index 52055e4a16..a843e059a1 100644
--- a/conversion/plamo.py
+++ b/conversion/plamo.py
@@ -64,7 +64,7 @@ class Plamo2Model(TextModel):
model_arch = gguf.MODEL_ARCH.PLAMO2
def set_vocab(self):
- self._set_vocab_plamo()
+ self._set_vocab_plamo(eot_token="<|plamo:op|>")
def set_gguf_parameters(self):
hparams = self.hparams
@@ -170,7 +170,10 @@ class Plamo3Model(TextModel):
})
def set_vocab(self):
- self._set_vocab_plamo()
+ self._set_vocab_plamo(
+ eot_token="<|plamo:tag|>",
+ normal_tokens=("<|plamo:begin_", "<|plamo:end_", ":plamo|>"),
+ )
tokenizer_config_path = self.dir_model / "tokenizer_config.json"
tokenizer_config = {}