Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(cpp): mutant-driven tokenizer tests, heap/rescan agreement, c2 fuzz CLI

Adeshina Lasisi7018246f785a
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txt
index 0fd7830d..77981fe5 100644
--- a/laya-cpp/CMakeLists.txt
+++ b/laya-cpp/CMakeLists.txt
@@ -53,6 +53,9 @@ if(LAYA_BUILD_TESTS)
add_executable(laya_cli tests/tools/laya_cli.cpp)
target_link_libraries(laya_cli PRIVATE laya)
laya_target_flags(laya_cli)
+ add_executable(laya_c2_cli tests/tools/laya_c2_cli.cpp)
+ target_link_libraries(laya_c2_cli PRIVATE laya)
+ laya_target_flags(laya_c2_cli)
include(GoogleTest)
# PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.
diff --git a/laya-cpp/include/laya/tokenizer.hpp b/laya-cpp/include/laya/tokenizer.hpp
index 72325e7c..c1c58749 100644
--- a/laya-cpp/include/laya/tokenizer.hpp
+++ b/laya-cpp/include/laya/tokenizer.hpp
@@ -60,6 +60,8 @@ class Tokenizer {
std::string MaskToken() const; // "" when none is configured
// Test seams: the stages of Encode, so a stage can be compared with the reference on its own.
+ // Pieces of text whose rescan and heap merges differ (must be 0); counts the pieces compared.
+ size_t CountMergeDisagreements(std::string_view text, size_t* pieces_checked) const;
std::string NormalizeForTest(std::string_view text) const;
std::vector<std::string> PreTokenizeForTest(std::string_view normalized) const;
diff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cpp
index 47443fa1..9171c055 100644
--- a/laya-cpp/src/tokenizer.cpp
+++ b/laya-cpp/src/tokenizer.cpp
@@ -1280,25 +1280,43 @@ class Bpe {
return;
}
}
- std::vector<Symbol> symbols;
- symbols.reserve(piece.size());
- for (size_t i = 0; i < piece.size();) {
- auto d = ud::DecodeRune(std::string_view(piece).substr(i));
- int32_t id;
- if (d.rune < kSmallRunes && d.rune != ud::kReplacement) {
- id = small_[d.rune];
- } else {
- auto it = vocab_->find(piece.substr(i, d.width));
- id = it == vocab_->end() ? -1 : it->second;
- }
- symbols.push_back(Symbol{int32_t(i), int32_t(i + d.width), id});
- i += d.width;
- }
+ std::vector<Symbol> symbols = Initial(piece);
if (symbols.size() >= kHeapMinSymbols) MergeWithHeap(symbols);
else MergeByRescan(symbols);
Emit(piece, symbols, out);
}
+ std::vector<Symbol> Initial(const std::string& piece) const {
+ std::vector<Symbol> symbols;
+ symbols.reserve(piece.size());
+ for (size_t i = 0; i < piece.size();) {
+ auto d = ud::DecodeRune(std::string_view(piece).substr(i));
+ int32_t id;
+ if (d.rune < kSmallRunes && d.rune != ud::kReplacement) {
+ id = small_[d.rune];
+ } else {
+ auto it = vocab_->find(piece.substr(i, d.width));
+ id = it == vocab_->end() ? -1 : it->second;
+ }
+ symbols.push_back(Symbol{int32_t(i), int32_t(i + d.width), id});
+ i += d.width;
+ }
+ return symbols;
+ }
+
+ // The two merge strategies must give one answer; the test seam compares them on a piece.
+ bool StrategiesAgree(const std::string& piece) const {
+ std::vector<Symbol> a = Initial(piece), b = a;
+ if (a.empty()) return true;
+ MergeByRescan(a);
+ MergeWithHeap(b);
+ if (a.size() != b.size()) return false;
+ for (size_t i = 0; i < a.size(); i++) {
+ if (a[i].start != b[i].start || a[i].end != b[i].end || a[i].id != b[i].id) return false;
+ }
+ return true;
+ }
+
// Both strategies are public for the test that they agree.
void MergeByRescan(std::vector<Symbol>& symbols) const {
std::vector<MergeRule> rules(symbols.size());
@@ -1722,6 +1740,17 @@ std::optional<int32_t> Tokenizer::Mask() const { return Role(*impl_, "mask_token
std::optional<int32_t> Tokenizer::Pad() const { return Role(*impl_, "pad_token"); }
std::string Tokenizer::MaskToken() const { return impl_->mask_token; }
+size_t Tokenizer::CountMergeDisagreements(std::string_view text, size_t* pieces_checked) const {
+ size_t bad = 0;
+ std::vector<std::string> pieces;
+ impl_->pre->Split(impl_->normalizer(ud::ToValid(text)), pieces);
+ for (const auto& p : pieces) {
+ if (!impl_->bpe->StrategiesAgree(p)) bad++;
+ if (pieces_checked != nullptr) (*pieces_checked)++;
+ }
+ return bad;
+}
+
std::string Tokenizer::NormalizeForTest(std::string_view text) const { return impl_->normalizer(ud::ToValid(text)); }
std::vector<std::string> Tokenizer::PreTokenizeForTest(std::string_view normalized) const {
std::vector<std::string> out;
diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cpp
index d29ce37e..4fee3949 100644
--- a/laya-cpp/tests/tokenizer_test.cpp
+++ b/laya-cpp/tests/tokenizer_test.cpp
@@ -1,5 +1,6 @@
// laya::tokenizer against the `tokenizers` crate (through the Python-recorded goldens) and against
// hand-built tokenizers. Ported from laya-go/tokenizer's tests.
+#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <mutex>
@@ -529,6 +530,7 @@ TEST(TokenizerAdded, LongestLstripRstripSingleWordNormalized) {
auto inside = tok.Encode("swords word.");
ASSERT_GE(inside.size(), 3u);
EXPECT_EQ(inside[inside.size() - 2], 303) << "a single_word token inside a word is not matched but a free one is";
+ EXPECT_EQ(std::count(inside.begin(), inside.end(), 303), 1) << "swords holds the word but not as a word";
EXPECT_EQ(tok.Encode("\xC3\xA9" "clair"), (V{304}));
EXPECT_EQ(tok.Encode("a\xC2\xA7" "b!"), (V{97, 305, 98, 306}));
EXPECT_EQ(tok.Encode("<S"), (V{60, 83})) << "a prefix of a token is not the token";
@@ -634,3 +636,65 @@ TEST(TokenizerPre, SequenceAndCornerCases) {
EXPECT_EQ(Join(got.value()), Join(c.want)) << c.in;
}
}
+
+// Short pieces merge by rescanning and long ones with a heap; both must give one answer on real
+// text, including pieces long enough for the heap.
+TEST_P(Shipped, HeapAndRescanMergeAgree) {
+ std::string name = GetParam();
+ LAYA_GOLDEN_LIST(side, "go_extra.json", "encode_fuzz", name.c_str());
+ LAYA_CHECKPOINT(tok, name);
+ size_t checked = 0, bad = 0;
+ for (const Value& t : side->Find("texts")->items()) {
+ bad += tok->CountMergeDisagreements(t.as_string(), &checked);
+ std::string longer; // a long piece: the heap path is taken from 32 symbols
+ for (char c : t.as_string()) {
+ if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) longer += c;
+ }
+ while (!longer.empty() && longer.size() < 200) longer += longer;
+ bad += tok->CountMergeDisagreements(longer, &checked);
+ }
+ EXPECT_EQ(bad, 0u);
+ EXPECT_GE(checked, 1000u);
+}
+
+TEST(TokenizerBpe, DuplicateMergeKeepsTheLowestRank) {
+ // "a b" is listed at rank 0 and again at rank 2, after "b c": the first listing decides, so abc -> ab, c.
+ Tokenizer tok = MustParse(
+ R"({"model":{"type":"BPE","vocab":{"a":0,"b":1,"c":2,"ab":3,"bc":4,"<unk>":5},"merges":["a b","b c","a b"],"unk_token":"<unk>"}})");
+ EXPECT_EQ(tok.Encode("abc"), (std::vector<int32_t>{3, 2}));
+}
+
+TEST(TokenizerBpe, ByteFallbackNeedsEveryByteAndResetsFusedUnknowns) {
+ const std::string head = R"({"model":{"type":"BPE","vocab":{"a":0,"<unk>":1,"<0xC3>":2,"<0xA9>":3},"merges":[],"unk_token":"<unk>","byte_fallback":true)";
+ using V = std::vector<int32_t>;
+ Tokenizer ok = MustParse(head + "}}");
+ EXPECT_EQ(ok.Encode("\xC3\xA9"), (V{2, 3})) << "every byte of the character resolves";
+ EXPECT_EQ(ok.Encode("\xC3\x80"), (V{1})) << "<0x80> is missing, so the whole character is unknown, not half bytes";
+ Tokenizer fused = MustParse(head + ",\"fuse_unk\":true}}");
+ EXPECT_EQ(fused.Encode("z\xC3\xA9z"), (V{1, 2, 3, 1})) << "bytes between two unknowns end the fused run";
+ EXPECT_EQ(fused.Encode("zzz"), (V{1}));
+}
+
+TEST(TokenizerRefusals, SequenceRunsInOrder) {
+ auto r = tk::RunNormalizerSpec(
+ R"({"type":"Sequence","normalizers":[{"type":"Prepend","prepend":"a"},{"type":"Replace","pattern":{"String":"a"},"content":"b"}]})", "a");
+ ASSERT_TRUE(r.ok());
+ EXPECT_EQ(r.value(), "bb");
+ auto s = tk::RunNormalizerSpec(
+ R"({"type":"Sequence","normalizers":[{"type":"Replace","pattern":{"String":"a"},"content":"b"},{"type":"Prepend","prepend":"a"}]})", "a");
+ ASSERT_TRUE(s.ok());
+ EXPECT_EQ(s.value(), "ab");
+}
+
+TEST(TokenizerAdded, RstripAbsorbsAllWhiteSpace) {
+ Tokenizer tok = MustParse(ByteLevelSpec(",\"added_tokens\":[{\"content\":\"[M]\",\"id\":302,\"special\":true,\"rstrip\":true}]"));
+ EXPECT_EQ(tok.Encode("[M] \t b"), (std::vector<int32_t>{302, 98}));
+}
+
+// Canonical ordering must run before composition: a + dot above (230) + dot below (220) is a-dot-below
+// then dot above (values recorded from the crate).
+TEST(TokenizerNormalize, MarksAreReorderedBeforeTheyCompose) {
+ EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFC), "\xE1\xBA\xA1\xCC\x87");
+ EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFKC), "\xE1\xBA\xA1\xCC\x87");
+ EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFD), "a\xCC\xA3\xCC\x87");
+}
diff --git a/laya-cpp/tests/tools/laya_c2_cli.cpp b/laya-cpp/tests/tools/laya_c2_cli.cpp
new file mode 100644
index 00000000..c6ffc5b8
--- /dev/null
+++ b/laya-cpp/tests/tools/laya_c2_cli.cpp
@@ -0,0 +1,92 @@
+// A tiny line protocol over laya::tokenizer for the differential fuzz (laya-e2e/cpp-lane/fuzz-c2):
+// argv[1] is the checkpoints directory; each stdin line is "<kind> <args...>" (bytes are hex), each
+// stdout line the answer.
+// enc <name> <hex> -> ids, space separated
+// dec <name> <skip 0|1> <ids> -> hex of the text (ids comma separated, "-" for none)
+// norm <NFC|NFD|NFKC|NFKD> <hex> -> hex lower <hex> -> hex
+// re <hex pattern> <hex content> <hex input> -> "refused" | "unsupported" | "ok <hex>"
+#include <iostream>
+#include <map>
+#include <sstream>
+
+#include "laya/tokenizer.hpp"
+
+namespace {
+std::string FromHex(const std::string& h) {
+ std::string out;
+ for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));
+ return out;
+}
+std::string Hex(const std::string& s) {
+ static const char* d = "0123456789abcdef";
+ std::string o;
+ for (unsigned char c : s) {
+ o.push_back(d[c >> 4]);
+ o.push_back(d[c & 15]);
+ }
+ return o;
+}
+// JSON string literal of raw bytes holding valid UTF-8 (the driver sends valid text only).
+std::string Quote(const std::string& s) {
+ std::string o = "\"";
+ for (unsigned char c : s) {
+ if (c == '"' || c == '\\') o += std::string("\\") + char(c);
+ else if (c < 0x20) { char b[8]; std::snprintf(b, sizeof b, "\\u%04x", c); o += b; }
+ else o += char(c);
+ }
+ return o + "\"";
+}
+} // namespace
+
+int main(int argc, char** argv) {
+ std::ios::sync_with_stdio(false);
+ if (argc < 2) return 2;
+ std::map<std::string, laya::tokenizer::Tokenizer> toks;
+ for (const char* name : {"english", "multilingual"}) {
+ auto t = laya::tokenizer::Tokenizer::LoadDir(std::string(argv[1]) + "/" + name);
+ if (!t.ok()) { std::cerr << t.error() << "\n"; return 2; }
+ toks.emplace(name, std::move(t).value());
+ }
+ std::string line;
+ while (std::getline(std::cin, line)) {
+ std::istringstream in(line);
+ std::string kind;
+ in >> kind;
+ if (kind == "enc") {
+ std::string name, h;
+ in >> name >> h;
+ std::string out;
+ for (int32_t id : toks[name].Encode(FromHex(h))) out += std::to_string(id) + " ";
+ std::cout << out << "\n";
+ } else if (kind == "dec") {
+ std::string name, ids;
+ int skip;
+ in >> name >> skip >> ids;
+ std::vector<int32_t> v;
+ std::istringstream is(ids == "-" ? "" : ids);
+ std::string tok;
+ while (std::getline(is, tok, ',')) v.push_back(int32_t(std::stol(tok)));
+ std::cout << Hex(toks[name].Decode(v, skip != 0)) << "\n";
+ } else if (kind == "norm") {
+ std::string form, h;
+ in >> form >> h;
+ auto f = form == "NFC" ? laya::tokenizer::Form::NFC : form == "NFD" ? laya::tokenizer::Form::NFD
+ : form == "NFKC" ? laya::tokenizer::Form::NFKC : laya::tokenizer::Form::NFKD;
+ std::cout << Hex(laya::tokenizer::Normalize(FromHex(h), f)) << "\n";
+ } else if (kind == "lower") {
+ std::string h;
+ in >> h;
+ std::cout << Hex(laya::tokenizer::Lowercase(FromHex(h))) << "\n";
+ } else if (kind == "re") {
+ std::string p, c, x;
+ in >> p >> c >> x;
+ std::string spec = "{\"type\":\"Replace\",\"pattern\":{\"Regex\":" + Quote(FromHex(p)) + "},\"content\":" + Quote(FromHex(c)) + "}";
+ auto r = laya::tokenizer::RunNormalizerSpec(spec, FromHex(x));
+ if (r.ok()) std::cout << "ok " << Hex(r.value()) << "\n";
+ else std::cout << (laya::tokenizer::IsUnsupported(r.error()) ? "unsupported" : "refused") << "\n";
+ } else {
+ std::cout << "bad\n";
+ }
+ }
+ return 0;
+}