LLM inference in C/C++
C++ ggml
unicode : reject overlong UTF-8 and out-of-range codepoints (#30485)
Simone Iacomino2ce20d22fdc1
diff --git a/src/unicode.cpp b/src/unicode.cppindex 8d8843e0e4..6c834cbced 100644--- a/src/unicode.cpp+++ b/src/unicode.cpp@@ -42,6 +42,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {throw std::invalid_argument("invalid character");}auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f);+ if (result < 0x80) {+ throw std::invalid_argument("invalid character");+ }offset += 2;return result;}@@ -50,6 +53,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {throw std::invalid_argument("invalid character");}auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f);+ if (result < 0x800) {+ throw std::invalid_argument("invalid character");+ }offset += 3;return result;}@@ -58,6 +64,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {throw std::invalid_argument("invalid character");}auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f);+ if (result < 0x10000 || result > 0x10ffff) {+ throw std::invalid_argument("invalid character");+ }offset += 4;return result;}diff --git a/tests/test-unicode.cpp b/tests/test-unicode.cppindex 561ff3e545..5ced7a28a7 100644--- a/tests/test-unicode.cpp+++ b/tests/test-unicode.cpp@@ -6,6 +6,7 @@#include <vector>int main() {+ bool success = true;const std::vector<std::pair<std::vector<uint32_t>, std::vector<uint32_t>>> normalization = {{{}, {}},{{0x65, 0x301}, {0xE9}},@@ -23,7 +24,46 @@ int main() {for (const auto & test : normalization) {if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) {fprintf(stderr, "unexpected NFC normalization\n");- return 1;+ success = false;+ break;+ }+ }++ const struct {+ std::string text;+ std::vector<uint32_t> expected;+ } utf8_tests[] = {+ { "\x7f", { 0x7f } },+ { "\xc2\x80", { 0x80 } },+ { "\xdf\xbf", { 0x7ff } },+ { "\xe0\xa0\x80", { 0x800 } },+ { "\xef\xbf\xbf", { 0xffff } },+ { "\xf0\x90\x80\x80", { 0x10000 } },+ { "\xf4\x8f\xbf\xbf", { 0x10ffff } },+ // Invalid-input cases preserve the existing recovery behavior: one U+FFFD per byte.+ { "\xc0\xa0", { 0xfffd, 0xfffd } },+ { "\xc1\xbf", { 0xfffd, 0xfffd } },+ { "\xe0\x80\xa0", { 0xfffd, 0xfffd, 0xfffd } },+ { "\xe0\x9f\xbf", { 0xfffd, 0xfffd, 0xfffd } },+ { "\xf0\x80\x80\xa0", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },+ { "\xf0\x8f\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },+ { "\xf4\x90\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },+ { "\xf5\x80\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },+ { "\xf7\xbf\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },+ { "\xe2\x80", { 0xfffd, 0xfffd } },+ };++ for (const auto & test : utf8_tests) {+ auto expected = test.expected;+ expected.insert(expected.begin(), 'a');+ expected.push_back('b');+ if (unicode_cpts_from_utf8("a" + test.text + "b") != expected) {+ fprintf(stderr, "unexpected UTF-8 decoding for bytes:");+ for (const unsigned char byte : test.text) {+ fprintf(stderr, " %02x", byte);+ }+ fprintf(stderr, "\n");+ success = false;}}@@ -39,8 +79,8 @@ int main() {fprintf(stderr, " [%s]", piece.c_str());}fprintf(stderr, "\n");- return 1;+ success = false;}- return 0;+ return success ? 0 : 1;}