Skip to content

LLM inference in C/C++

C++ ggml

vocab : validate plamo2 byte tokens (#26511)

Sigbjørn Skjæretcf06ad7dfe79
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 10032a8c64..4a01dfd4ca 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -1373,8 +1373,10 @@ struct llm_tokenizer_plamo2 : llm_tokenizer {
if (vocab.is_byte(token_id)) {
if (entry.text.length() == 6 && entry.text.substr(0, 3) == "<0x" && entry.text.back() == '>') {
std::string hex_str = entry.text.substr(3, 2);
- int byte_val = std::stoi(hex_str, nullptr, 16);
- bytes_[byte_val] = static_cast<llama_token>(token_id);
+ if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {
+ int byte_val = std::stoi(hex_str, nullptr, 16);
+ bytes_[byte_val] = static_cast<llama_token>(token_id);
+ }
}
continue;
}
@@ -3625,12 +3627,15 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
if (vocab.is_byte(token)) {
// Handle byte tokens like <0xXX>
if (token_text.length() == 6 && token_text.substr(0, 3) == "<0x" && token_text.back() == '>') {
- int hex_val = std::stoi(token_text.substr(3, 2), nullptr, 16);
- if (length < 1) {
- return -1;
+ std::string hex_str = token_text.substr(3, 2);
+ if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {
+ int hex_val = std::stoi(hex_str, nullptr, 16);
+ if (length < 1) {
+ return -1;
+ }
+ buf[0] = static_cast<char>(hex_val);
+ return 1;
}
- buf[0] = static_cast<char>(hex_val);
- return 1;
}
}