Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(cpp): mutant-driven tokenizer tests, heap/rescan agreement, c2 fuzz CLI
Adeshina Lasisi7018246f785a
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txtindex 0fd7830d..77981fe5 100644--- a/laya-cpp/CMakeLists.txt+++ b/laya-cpp/CMakeLists.txt@@ -53,6 +53,9 @@ if(LAYA_BUILD_TESTS)add_executable(laya_cli tests/tools/laya_cli.cpp)target_link_libraries(laya_cli PRIVATE laya)laya_target_flags(laya_cli)+ add_executable(laya_c2_cli tests/tools/laya_c2_cli.cpp)+ target_link_libraries(laya_c2_cli PRIVATE laya)+ laya_target_flags(laya_c2_cli)include(GoogleTest)# PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.diff --git a/laya-cpp/include/laya/tokenizer.hpp b/laya-cpp/include/laya/tokenizer.hppindex 72325e7c..c1c58749 100644--- a/laya-cpp/include/laya/tokenizer.hpp+++ b/laya-cpp/include/laya/tokenizer.hpp@@ -60,6 +60,8 @@ class Tokenizer {std::string MaskToken() const; // "" when none is configured// Test seams: the stages of Encode, so a stage can be compared with the reference on its own.+ // Pieces of text whose rescan and heap merges differ (must be 0); counts the pieces compared.+ size_t CountMergeDisagreements(std::string_view text, size_t* pieces_checked) const;std::string NormalizeForTest(std::string_view text) const;std::vector<std::string> PreTokenizeForTest(std::string_view normalized) const;diff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cppindex 47443fa1..9171c055 100644--- a/laya-cpp/src/tokenizer.cpp+++ b/laya-cpp/src/tokenizer.cpp@@ -1280,25 +1280,43 @@ class Bpe {return;}}- std::vector<Symbol> symbols;- symbols.reserve(piece.size());- for (size_t i = 0; i < piece.size();) {- auto d = ud::DecodeRune(std::string_view(piece).substr(i));- int32_t id;- if (d.rune < kSmallRunes && d.rune != ud::kReplacement) {- id = small_[d.rune];- } else {- auto it = vocab_->find(piece.substr(i, d.width));- id = it == vocab_->end() ? -1 : it->second;- }- symbols.push_back(Symbol{int32_t(i), int32_t(i + d.width), id});- i += d.width;- }+ std::vector<Symbol> symbols = Initial(piece);if (symbols.size() >= kHeapMinSymbols) MergeWithHeap(symbols);else MergeByRescan(symbols);Emit(piece, symbols, out);}+ std::vector<Symbol> Initial(const std::string& piece) const {+ std::vector<Symbol> symbols;+ symbols.reserve(piece.size());+ for (size_t i = 0; i < piece.size();) {+ auto d = ud::DecodeRune(std::string_view(piece).substr(i));+ int32_t id;+ if (d.rune < kSmallRunes && d.rune != ud::kReplacement) {+ id = small_[d.rune];+ } else {+ auto it = vocab_->find(piece.substr(i, d.width));+ id = it == vocab_->end() ? -1 : it->second;+ }+ symbols.push_back(Symbol{int32_t(i), int32_t(i + d.width), id});+ i += d.width;+ }+ return symbols;+ }++ // The two merge strategies must give one answer; the test seam compares them on a piece.+ bool StrategiesAgree(const std::string& piece) const {+ std::vector<Symbol> a = Initial(piece), b = a;+ if (a.empty()) return true;+ MergeByRescan(a);+ MergeWithHeap(b);+ if (a.size() != b.size()) return false;+ for (size_t i = 0; i < a.size(); i++) {+ if (a[i].start != b[i].start || a[i].end != b[i].end || a[i].id != b[i].id) return false;+ }+ return true;+ }+// Both strategies are public for the test that they agree.void MergeByRescan(std::vector<Symbol>& symbols) const {std::vector<MergeRule> rules(symbols.size());@@ -1722,6 +1740,17 @@ std::optional<int32_t> Tokenizer::Mask() const { return Role(*impl_, "mask_tokenstd::optional<int32_t> Tokenizer::Pad() const { return Role(*impl_, "pad_token"); }std::string Tokenizer::MaskToken() const { return impl_->mask_token; }+size_t Tokenizer::CountMergeDisagreements(std::string_view text, size_t* pieces_checked) const {+ size_t bad = 0;+ std::vector<std::string> pieces;+ impl_->pre->Split(impl_->normalizer(ud::ToValid(text)), pieces);+ for (const auto& p : pieces) {+ if (!impl_->bpe->StrategiesAgree(p)) bad++;+ if (pieces_checked != nullptr) (*pieces_checked)++;+ }+ return bad;+}+std::string Tokenizer::NormalizeForTest(std::string_view text) const { return impl_->normalizer(ud::ToValid(text)); }std::vector<std::string> Tokenizer::PreTokenizeForTest(std::string_view normalized) const {std::vector<std::string> out;diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cppindex d29ce37e..4fee3949 100644--- a/laya-cpp/tests/tokenizer_test.cpp+++ b/laya-cpp/tests/tokenizer_test.cpp@@ -1,5 +1,6 @@// laya::tokenizer against the `tokenizers` crate (through the Python-recorded goldens) and against// hand-built tokenizers. Ported from laya-go/tokenizer's tests.+#include <algorithm>#include <cstdlib>#include <fstream>#include <mutex>@@ -529,6 +530,7 @@ TEST(TokenizerAdded, LongestLstripRstripSingleWordNormalized) {auto inside = tok.Encode("swords word.");ASSERT_GE(inside.size(), 3u);EXPECT_EQ(inside[inside.size() - 2], 303) << "a single_word token inside a word is not matched but a free one is";+ EXPECT_EQ(std::count(inside.begin(), inside.end(), 303), 1) << "swords holds the word but not as a word";EXPECT_EQ(tok.Encode("\xC3\xA9" "clair"), (V{304}));EXPECT_EQ(tok.Encode("a\xC2\xA7" "b!"), (V{97, 305, 98, 306}));EXPECT_EQ(tok.Encode("<S"), (V{60, 83})) << "a prefix of a token is not the token";@@ -634,3 +636,65 @@ TEST(TokenizerPre, SequenceAndCornerCases) {EXPECT_EQ(Join(got.value()), Join(c.want)) << c.in;}}++// Short pieces merge by rescanning and long ones with a heap; both must give one answer on real+// text, including pieces long enough for the heap.+TEST_P(Shipped, HeapAndRescanMergeAgree) {+ std::string name = GetParam();+ LAYA_GOLDEN_LIST(side, "go_extra.json", "encode_fuzz", name.c_str());+ LAYA_CHECKPOINT(tok, name);+ size_t checked = 0, bad = 0;+ for (const Value& t : side->Find("texts")->items()) {+ bad += tok->CountMergeDisagreements(t.as_string(), &checked);+ std::string longer; // a long piece: the heap path is taken from 32 symbols+ for (char c : t.as_string()) {+ if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) longer += c;+ }+ while (!longer.empty() && longer.size() < 200) longer += longer;+ bad += tok->CountMergeDisagreements(longer, &checked);+ }+ EXPECT_EQ(bad, 0u);+ EXPECT_GE(checked, 1000u);+}++TEST(TokenizerBpe, DuplicateMergeKeepsTheLowestRank) {+ // "a b" is listed at rank 0 and again at rank 2, after "b c": the first listing decides, so abc -> ab, c.+ Tokenizer tok = MustParse(+ R"({"model":{"type":"BPE","vocab":{"a":0,"b":1,"c":2,"ab":3,"bc":4,"<unk>":5},"merges":["a b","b c","a b"],"unk_token":"<unk>"}})");+ EXPECT_EQ(tok.Encode("abc"), (std::vector<int32_t>{3, 2}));+}++TEST(TokenizerBpe, ByteFallbackNeedsEveryByteAndResetsFusedUnknowns) {+ const std::string head = R"({"model":{"type":"BPE","vocab":{"a":0,"<unk>":1,"<0xC3>":2,"<0xA9>":3},"merges":[],"unk_token":"<unk>","byte_fallback":true)";+ using V = std::vector<int32_t>;+ Tokenizer ok = MustParse(head + "}}");+ EXPECT_EQ(ok.Encode("\xC3\xA9"), (V{2, 3})) << "every byte of the character resolves";+ EXPECT_EQ(ok.Encode("\xC3\x80"), (V{1})) << "<0x80> is missing, so the whole character is unknown, not half bytes";+ Tokenizer fused = MustParse(head + ",\"fuse_unk\":true}}");+ EXPECT_EQ(fused.Encode("z\xC3\xA9z"), (V{1, 2, 3, 1})) << "bytes between two unknowns end the fused run";+ EXPECT_EQ(fused.Encode("zzz"), (V{1}));+}++TEST(TokenizerRefusals, SequenceRunsInOrder) {+ auto r = tk::RunNormalizerSpec(+ R"({"type":"Sequence","normalizers":[{"type":"Prepend","prepend":"a"},{"type":"Replace","pattern":{"String":"a"},"content":"b"}]})", "a");+ ASSERT_TRUE(r.ok());+ EXPECT_EQ(r.value(), "bb");+ auto s = tk::RunNormalizerSpec(+ R"({"type":"Sequence","normalizers":[{"type":"Replace","pattern":{"String":"a"},"content":"b"},{"type":"Prepend","prepend":"a"}]})", "a");+ ASSERT_TRUE(s.ok());+ EXPECT_EQ(s.value(), "ab");+}++TEST(TokenizerAdded, RstripAbsorbsAllWhiteSpace) {+ Tokenizer tok = MustParse(ByteLevelSpec(",\"added_tokens\":[{\"content\":\"[M]\",\"id\":302,\"special\":true,\"rstrip\":true}]"));+ EXPECT_EQ(tok.Encode("[M] \t b"), (std::vector<int32_t>{302, 98}));+}++// Canonical ordering must run before composition: a + dot above (230) + dot below (220) is a-dot-below+// then dot above (values recorded from the crate).+TEST(TokenizerNormalize, MarksAreReorderedBeforeTheyCompose) {+ EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFC), "\xE1\xBA\xA1\xCC\x87");+ EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFKC), "\xE1\xBA\xA1\xCC\x87");+ EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFD), "a\xCC\xA3\xCC\x87");+}diff --git a/laya-cpp/tests/tools/laya_c2_cli.cpp b/laya-cpp/tests/tools/laya_c2_cli.cppnew file mode 100644index 00000000..c6ffc5b8--- /dev/null+++ b/laya-cpp/tests/tools/laya_c2_cli.cpp@@ -0,0 +1,92 @@+// A tiny line protocol over laya::tokenizer for the differential fuzz (laya-e2e/cpp-lane/fuzz-c2):+// argv[1] is the checkpoints directory; each stdin line is "<kind> <args...>" (bytes are hex), each+// stdout line the answer.+// enc <name> <hex> -> ids, space separated+// dec <name> <skip 0|1> <ids> -> hex of the text (ids comma separated, "-" for none)+// norm <NFC|NFD|NFKC|NFKD> <hex> -> hex lower <hex> -> hex+// re <hex pattern> <hex content> <hex input> -> "refused" | "unsupported" | "ok <hex>"+#include <iostream>+#include <map>+#include <sstream>++#include "laya/tokenizer.hpp"++namespace {+std::string FromHex(const std::string& h) {+ std::string out;+ for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(char(std::stoi(h.substr(i, 2), nullptr, 16)));+ return out;+}+std::string Hex(const std::string& s) {+ static const char* d = "0123456789abcdef";+ std::string o;+ for (unsigned char c : s) {+ o.push_back(d[c >> 4]);+ o.push_back(d[c & 15]);+ }+ return o;+}+// JSON string literal of raw bytes holding valid UTF-8 (the driver sends valid text only).+std::string Quote(const std::string& s) {+ std::string o = "\"";+ for (unsigned char c : s) {+ if (c == '"' || c == '\\') o += std::string("\\") + char(c);+ else if (c < 0x20) { char b[8]; std::snprintf(b, sizeof b, "\\u%04x", c); o += b; }+ else o += char(c);+ }+ return o + "\"";+}+} // namespace++int main(int argc, char** argv) {+ std::ios::sync_with_stdio(false);+ if (argc < 2) return 2;+ std::map<std::string, laya::tokenizer::Tokenizer> toks;+ for (const char* name : {"english", "multilingual"}) {+ auto t = laya::tokenizer::Tokenizer::LoadDir(std::string(argv[1]) + "/" + name);+ if (!t.ok()) { std::cerr << t.error() << "\n"; return 2; }+ toks.emplace(name, std::move(t).value());+ }+ std::string line;+ while (std::getline(std::cin, line)) {+ std::istringstream in(line);+ std::string kind;+ in >> kind;+ if (kind == "enc") {+ std::string name, h;+ in >> name >> h;+ std::string out;+ for (int32_t id : toks[name].Encode(FromHex(h))) out += std::to_string(id) + " ";+ std::cout << out << "\n";+ } else if (kind == "dec") {+ std::string name, ids;+ int skip;+ in >> name >> skip >> ids;+ std::vector<int32_t> v;+ std::istringstream is(ids == "-" ? "" : ids);+ std::string tok;+ while (std::getline(is, tok, ',')) v.push_back(int32_t(std::stol(tok)));+ std::cout << Hex(toks[name].Decode(v, skip != 0)) << "\n";+ } else if (kind == "norm") {+ std::string form, h;+ in >> form >> h;+ auto f = form == "NFC" ? laya::tokenizer::Form::NFC : form == "NFD" ? laya::tokenizer::Form::NFD+ : form == "NFKC" ? laya::tokenizer::Form::NFKC : laya::tokenizer::Form::NFKD;+ std::cout << Hex(laya::tokenizer::Normalize(FromHex(h), f)) << "\n";+ } else if (kind == "lower") {+ std::string h;+ in >> h;+ std::cout << Hex(laya::tokenizer::Lowercase(FromHex(h))) << "\n";+ } else if (kind == "re") {+ std::string p, c, x;+ in >> p >> c >> x;+ std::string spec = "{\"type\":\"Replace\",\"pattern\":{\"Regex\":" + Quote(FromHex(p)) + "},\"content\":" + Quote(FromHex(c)) + "}";+ auto r = laya::tokenizer::RunNormalizerSpec(spec, FromHex(x));+ if (r.ok()) std::cout << "ok " << Hex(r.value()) << "\n";+ else std::cout << (laya::tokenizer::IsUnsupported(r.error()) ? "unsupported" : "refused") << "\n";+ } else {+ std::cout << "bad\n";+ }+ }+ return 0;+}