LLM inference in C/C++
C++ ggml
vocab : validate plamo2 byte tokens (#26511)
Sigbjørn Skjæretcf06ad7dfe79
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cppindex 10032a8c64..4a01dfd4ca 100644--- a/src/llama-vocab.cpp+++ b/src/llama-vocab.cpp@@ -1373,8 +1373,10 @@ struct llm_tokenizer_plamo2 : llm_tokenizer {if (vocab.is_byte(token_id)) {if (entry.text.length() == 6 && entry.text.substr(0, 3) == "<0x" && entry.text.back() == '>') {std::string hex_str = entry.text.substr(3, 2);- int byte_val = std::stoi(hex_str, nullptr, 16);- bytes_[byte_val] = static_cast<llama_token>(token_id);+ if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {+ int byte_val = std::stoi(hex_str, nullptr, 16);+ bytes_[byte_val] = static_cast<llama_token>(token_id);+ }}continue;}@@ -3625,12 +3627,15 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_tif (vocab.is_byte(token)) {// Handle byte tokens like <0xXX>if (token_text.length() == 6 && token_text.substr(0, 3) == "<0x" && token_text.back() == '>') {- int hex_val = std::stoi(token_text.substr(3, 2), nullptr, 16);- if (length < 1) {- return -1;+ std::string hex_str = token_text.substr(3, 2);+ if (std::isxdigit(static_cast<unsigned char>(hex_str[0])) && std::isxdigit(static_cast<unsigned char>(hex_str[1]))) {+ int hex_val = std::stoi(hex_str, nullptr, 16);+ if (length < 1) {+ return -1;+ }+ buf[0] = static_cast<char>(hex_val);+ return 1;}- buf[0] = static_cast<char>(hex_val);- return 1;}}