Skip to content

LLM inference in C/C++

C++ ggml

unicode : include '~' in collapsed symbol class (#26972)

Thiago Padilhaafd439df1f08
diff --git a/src/unicode.cpp b/src/unicode.cpp
index b02ecdc930..93996f9dd5 100644
--- a/src/unicode.cpp
+++ b/src/unicode.cpp
@@ -1241,7 +1241,7 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
{ unicode_cpt_flags::LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z
{ unicode_cpt_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}
{ unicode_cpt_flags::ACCENT_MARK, "" }, // no sub-128 codepoints
- { unicode_cpt_flags::SYMBOL, "\\\x24\\\x2B\x3C-\x3E\x5E\x60\\\x7C" }, // $+<=>^`|
+ { unicode_cpt_flags::SYMBOL, "\\\x24\\\x2B\x3C-\x3E\x5E\x60\\\x7C\\\x7E" }, // $+<=>^`|~
};
// compute collapsed codepoints only if needed by at least one regex
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 3ee51b5191..0db7fd9ef3 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -116,6 +116,8 @@ function(llama_build_and_test source)
set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL})
endfunction()
+llama_build_and_test(test-unicode.cpp)
+
# build test-tokenizer-0 target once and add many tests
llama_build(test-tokenizer-0.cpp)
diff --git a/tests/test-unicode.cpp b/tests/test-unicode.cpp
new file mode 100644
index 0000000000..2347d9000a
--- /dev/null
+++ b/tests/test-unicode.cpp
@@ -0,0 +1,24 @@
+#include "../src/unicode.h"
+
+#include <cstdio>
+#include <string>
+#include <vector>
+
+int main() {
+ const std::vector<std::string> regex_exprs = {
+ "[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
+ };
+ const std::vector<std::string> expected = { " ~", "foo" };
+ const auto actual = unicode_regex_split(" ~foo", regex_exprs, false);
+
+ if (actual != expected) {
+ fprintf(stderr, "unexpected split:");
+ for (const auto & piece : actual) {
+ fprintf(stderr, " [%s]", piece.c_str());
+ }
+ fprintf(stderr, "\n");
+ return 1;
+ }
+
+ return 0;
+}