Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(cpp): tokenizer review round 1
Adeshina Lasisif0eadf680e4f
diff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cppindex 94a00c3a..01a1e06e 100644--- a/laya-cpp/src/tokenizer.cpp+++ b/laya-cpp/src/tokenizer.cpp@@ -282,9 +282,10 @@ class Regex {std::string ReplaceAll(const std::string& text, const std::string& content) const {std::string out;size_t copied = 0, from = 0;+ Memo memo(prog_.size(), text.size() + 1);while (from <= text.size()) {size_t ms, me;- if (!Find(text, from, &ms, &me)) break;+ if (!Find(text, from, &ms, &me, memo)) break;out.append(text, copied, ms - copied);out += content;copied = me;@@ -295,7 +296,8 @@ class Regex {}bool MatchesEmpty() const {size_t s, e;- return Find("", 0, &s, &e);+ Memo memo(prog_.size(), 1);+ return Find("", 0, &s, &e, memo);}private:@@ -312,6 +314,36 @@ class Regex {std::vector<Inst> prog_;std::vector<Set> sets_;+ static constexpr size_t kMaxProgram = 100000; // instructions after repeats are expanded+ static constexpr int kMaxDepth = 256; // nested groups+ static constexpr size_t kDenseBits = size_t(1) << 28;+ int depth_ = 0;++ // The (instruction, position) states already tried. Dense while the table is small, a hash set above+ // that. One table serves a whole ReplaceAll: a state that failed still fails for any later start, and+ // the states of a match lie at or before its end, which no later (non-empty) match revisits.+ class Memo {+ public:+ Memo(size_t insts, size_t positions) : insts_(insts), dense_(insts * positions <= kDenseBits) {+ if (dense_) bits_.assign((insts * positions + 63) / 64, 0);+ }+ bool Visit(int pc, size_t pos) { // true the first time+ size_t key = pos * insts_ + size_t(pc);+ if (!dense_) return sparse_.insert(key).second;+ uint64_t& w = bits_[key / 64];+ uint64_t bit = uint64_t(1) << (key % 64);+ if (w & bit) return false;+ w |= bit;+ return true;+ }++ private:+ size_t insts_;+ bool dense_;+ std::vector<uint64_t> bits_;+ std::unordered_set<size_t> sparse_;+ };+// --- parser: pattern -> instructions, emitted directly from a recursive descent -------------std::vector<char32_t> p_;size_t at_ = 0;@@ -320,6 +352,7 @@ class Regex {bool More() const { return at_ < p_.size(); }char32_t Peek() const { return p_[at_]; }int Emit(Op op, char32_t c = 0, int x = 0, int y = 0) {+ if (prog_.size() >= kMaxProgram) Refuse("a Replace regex that compiles to more than 100000 instructions");prog_.push_back(Inst{op, c, x, y});return int(prog_.size()) - 1;}@@ -341,34 +374,24 @@ class Regex {}}+ // Each branch starts with a placeholder split (a plain jump for the last one), so no instruction is+ // ever inserted and every target stays valid.void Alternation() {- int start = int(prog_.size());- Concatenation();std::vector<int> jumps;- while (More() && Peek() == '|') {- at_++;- // split(start of the left branches, next), the left branch jumps over the right.- prog_.insert(prog_.begin() + start, Inst{kSplit, 0, 0, 0});- Shift(start, 1);- int j = Emit(kJmp);- jumps.push_back(j);- prog_[size_t(start)].x = start + 1;- prog_[size_t(start)].y = int(prog_.size());+ while (true) {+ int split = Emit(kSplit);Concatenation();- }- for (int j : jumps) prog_[size_t(j)].x = int(prog_.size());- }-- // Inserting an instruction at `from` moves everything after it, so targets at or past `from`- // that were emitted before the insertion move by `by`.- void Shift(int from, int by) {- for (size_t i = size_t(from) + 1; i < prog_.size(); i++) { // [from] is the inserted instruction- Inst& in = prog_[i];- if (in.op == kSplit || in.op == kJmp) {- if (in.x >= from) in.x += by;- if (in.op == kSplit && in.y >= from) in.y += by;+ if (More() && Peek() == '|') {+ at_++;+ jumps.push_back(Emit(kJmp)); // the finished branch jumps over the rest+ prog_[size_t(split)].x = split + 1;+ prog_[size_t(split)].y = int(prog_.size());+ continue;}+ prog_[size_t(split)] = Inst{kJmp, 0, split + 1, 0};+ break;}+ for (int j : jumps) prog_[size_t(j)].x = int(prog_.size());}void Concatenation() {@@ -406,6 +429,7 @@ class Regex {prog_.resize(size_t(start));if (lo > 1000 || hi > 1000) Refuse("a Replace regex repeat count above 1000");auto copy = [&]() {+ if (prog_.size() + body.size() > kMaxProgram) Refuse("a Replace regex that compiles to more than 100000 instructions");int base = int(prog_.size());for (Inst in : body) {if (in.op == kSplit || in.op == kJmp) {@@ -492,7 +516,9 @@ class Regex {if (at_ + 1 < p_.size() && p_[at_ + 1] == ':') at_ += 2;else Unsupported("this Replace regex group syntax (?...) is not implemented");}+ if (++depth_ > kMaxDepth) Refuse("a Replace regex nested more than 256 groups deep");Alternation();+ depth_--;if (!More() || Peek() != ')') Refuse("a Replace normalizer's regex: missing )");at_++;} else if (c == '[') {@@ -533,7 +559,17 @@ class Regex {Unsupported("the Replace regex escape \\" + std::string(1, char(c < 0x80 ? c : '?')) + " is not implemented");}+ // Oniguruma folds with full Unicode rules: K (Kelvin) is k, the long s and the sharp s are s/ss, and+ // the ligatures ff, fi, fl, ffi, ffl, st fold to letter pairs. An ASCII-only fold would differ.+ void FoldTrap(char32_t lo, char32_t hi) const {+ if (!icase_) return;+ for (char32_t t : {U'k', U'K', U's', U'S', U'f', U'F'}) {+ if (lo <= t && t <= hi) Unsupported("a case-insensitive Replace regex with k, s or f, whose Unicode case folding is not implemented");+ }+ }+void Literal(char32_t c) {+ FoldTrap(c, c);if (icase_ && ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) {Set s;s.ranges = {{c | 0x20, c | 0x20}, {c & ~char32_t(0x20), c & ~char32_t(0x20)}};@@ -567,6 +603,7 @@ class Regex {hi = h == '\\' ? Escape() : (at_++, h);if (hi < lo) Refuse("a Replace normalizer's regex: a range with its ends reversed");}+ FoldTrap(lo, hi);s.ranges.push_back({lo, hi});}if (icase_) {@@ -585,9 +622,7 @@ class Regex {}// --- matcher --------------------------------------------------------------------------------- bool Find(const std::string& text, size_t from, size_t* ms, size_t* me) const {- size_t width = text.size() + 1;- std::vector<bool> seen(prog_.size() * width, false);+ bool Find(const std::string& text, size_t from, size_t* ms, size_t* me, Memo& seen) const {std::vector<std::pair<int, size_t>> stack;for (size_t start = from; start <= text.size();) {stack.assign(1, {0, start});@@ -595,9 +630,7 @@ class Regex {auto [pc, pos] = stack.back();stack.pop_back();while (true) {- size_t key = size_t(pc) * width + pos;- if (seen[key]) break;- seen[key] = true;+ if (!seen.Visit(pc, pos)) break;const Inst& in = prog_[size_t(pc)];if (in.op == kMatch) {*ms = start;@@ -707,7 +740,7 @@ std::string ReplaceAllLiteral(const std::string& s, const std::string& from, conreturn out;}-Normalizer ParseNormalizer(const Value* n) {+Normalizer ParseNormalizer(const Value* n, bool* may_drop = nullptr) {if (n == nullptr || n->is_null()) return [](const std::string& s) { return s; };std::string kind = TypeOf(*n, "normalizer");if (kind == "NFC") return [](const std::string& s) { return NormalizeImpl(s, Form::NFC); };@@ -721,9 +754,11 @@ Normalizer ParseNormalizer(const Value* n) {}if (kind == "Strip") {bool l = Bool(*n, "strip_left", true), r = Bool(*n, "strip_right", true);+ if (l && may_drop != nullptr) *may_drop = true;return [l, r](const std::string& s) { return StripImpl(s, l, r); };}if (kind == "Replace") {+ if (may_drop != nullptr) *may_drop = true;const Value& pattern = Object(*n, "pattern");const Value* content = n->Find("content");if (content == nullptr || content->kind() != Value::Kind::String) Refuse("a Replace normalizer must declare content");@@ -750,7 +785,7 @@ Normalizer ParseNormalizer(const Value* n) {std::vector<Normalizer> steps;for (const Value& child : Array(*n, "normalizers").items()) {if (child.kind() != Value::Kind::Object) Refuse("a normalizer in a Sequence must be an object");- steps.push_back(ParseNormalizer(&child));+ steps.push_back(ParseNormalizer(&child, may_drop));}return [steps](const std::string& s) {std::string cur = s;@@ -792,12 +827,14 @@ const Alphabet& TheAlphabet() {class PreTokenizer {public:virtual ~PreTokenizer() = default;- virtual void Split(std::string_view text, std::vector<std::string>& out) const = 0;+ // `origin`: the text starts where the input did (not after an added token).+ virtual void Split(std::string_view text, std::vector<std::string>& out, bool origin) const = 0;+ virtual bool PrependsFirstOnly() const { return false; }};class WholeText : public PreTokenizer {public:- void Split(std::string_view text, std::vector<std::string>& out) const override {+ void Split(std::string_view text, std::vector<std::string>& out, bool) const override {if (!text.empty()) out.emplace_back(text);}};@@ -805,7 +842,7 @@ class WholeText : public PreTokenizer {class ByteLevel : public PreTokenizer {public:ByteLevel(bool prefix, bool regex) : add_prefix_space_(prefix), use_regex_(regex) {}- void Split(std::string_view text, std::vector<std::string>& out) const override {+ void Split(std::string_view text, std::vector<std::string>& out, bool) const override {if (text.empty()) return;std::string source(text);if (add_prefix_space_ && source[0] != ' ') source = " " + source;@@ -874,12 +911,13 @@ class ByteLevel : public PreTokenizer {class Metaspace : public PreTokenizer {public:- Metaspace(std::string replacement, bool prepend, bool split)- : replacement_(std::move(replacement)), prepend_(prepend), split_(split) {}- void Split(std::string_view text, std::vector<std::string>& out) const override {+ Metaspace(std::string replacement, bool prepend, bool first_only, bool split)+ : replacement_(std::move(replacement)), prepend_(prepend), first_only_(first_only), split_(split) {}+ bool PrependsFirstOnly() const override { return prepend_ && first_only_; }+ void Split(std::string_view text, std::vector<std::string>& out, bool origin) const override {if (text.empty()) return;std::string source = ReplaceAllLiteral(std::string(text), " ", replacement_);- if (prepend_ && source.compare(0, replacement_.size(), replacement_) != 0) source = replacement_ + source;+ if (prepend_ && (origin || !first_only_) && source.compare(0, replacement_.size(), replacement_) != 0) source = replacement_ + source;if (!split_) {out.push_back(source);return;@@ -898,17 +936,24 @@ class Metaspace : public PreTokenizer {private:std::string replacement_;- bool prepend_, split_;+ bool prepend_, first_only_, split_;};class SequencePre : public PreTokenizer {public:std::vector<std::unique_ptr<PreTokenizer>> steps;- void Split(std::string_view text, std::vector<std::string>& out) const override {+ bool PrependsFirstOnly() const override {+ for (const auto& step : steps) {+ if (step->PrependsFirstOnly()) return true;+ }+ return false;+ }+ void Split(std::string_view text, std::vector<std::string>& out, bool origin) const override {std::vector<std::string> current{std::string(text)};for (const auto& step : steps) {std::vector<std::string> next;- for (const auto& piece : current) step->Split(piece, next);+ // Every pre-tokenizer here keeps the text whole, so only the first piece is still at the start.+ for (size_t i = 0; i < current.size(); i++) step->Split(current[i], next, origin && i == 0);current.swap(next);}out.insert(out.end(), current.begin(), current.end());@@ -924,8 +969,13 @@ std::unique_ptr<PreTokenizer> ParsePreTokenizer(const Value* n) {if (ud::RuneCount(replacement) != 1) Refuse("a Metaspace replacement must be exactly one code point");std::string scheme = Str(*n, "prepend_scheme", "always");bool split = Bool(*n, "split", true);- if (scheme == "always" || scheme == "first") return std::make_unique<Metaspace>(replacement, true, split);- if (scheme == "never") return std::make_unique<Metaspace>(replacement, false, split);+ // The legacy flag must agree with the scheme, as in the crate (true with "never" is accepted).+ if (!Bool(*n, "add_prefix_space", true) && scheme != "never") {+ Refuse("a Metaspace add_prefix_space does not match its declared prepend_scheme");+ }+ if (scheme == "always") return std::make_unique<Metaspace>(replacement, true, false, split);+ if (scheme == "first") return std::make_unique<Metaspace>(replacement, true, true, split);+ if (scheme == "never") return std::make_unique<Metaspace>(replacement, false, false, split);Refuse("unknown Metaspace prepend_scheme \"" + scheme + "\"");}if (kind == "Sequence") {@@ -1124,6 +1174,7 @@ struct Span {std::string text;int32_t id = 0;bool is_token = false;+ size_t offset = 0; // byte offset of the span in the text that was cut};// The added tokens that start with one character, longest first (the reference's leftmost-longest@@ -1166,13 +1217,18 @@ class AddedVocab {std::vector<Span> out;if (text.empty()) return out;if (empty) {- out.push_back(Span{text, 0, false});+ out.push_back(Span{text, 0, false, 0});return out;}size_t plain_from = 0;for (size_t at = 0; at < text.size();) {auto d = ud::DecodeRune(std::string_view(text).substr(at));- const AddedToken* hit = LongestAt(text, at, d.rune, d.width);+ size_t skip = 0;+ const AddedToken* hit = LongestAt(text, at, d.rune, d.width, &skip);+ if (skip != 0) { // the crate skips a longest hit that fails single_word, shorter tokens inside it too+ at += skip;+ continue;+ }if (hit == nullptr) {at += d.width;continue;@@ -1192,11 +1248,11 @@ class AddedVocab {stop += nx.width;}}- if (start > plain_from) out.push_back(Span{text.substr(plain_from, start - plain_from), 0, false});- out.push_back(Span{hit->content, hit->id, true});+ if (start > plain_from) out.push_back(Span{text.substr(plain_from, start - plain_from), 0, false, plain_from});+ out.push_back(Span{hit->content, hit->id, true, start});plain_from = at = stop;}- if (plain_from < text.size()) out.push_back(Span{text.substr(plain_from), 0, false});+ if (plain_from < text.size()) out.push_back(Span{text.substr(plain_from), 0, false, plain_from});return out;}@@ -1206,7 +1262,8 @@ class AddedVocab {auto it = other_.find(r);return it == other_.end() ? nullptr : &it->second;}- const AddedToken* LongestAt(const std::string& text, size_t at, char32_t first, size_t width) const {+ // `skip` is set to the length of a longest hit that fails single_word.+ const AddedToken* LongestAt(const std::string& text, size_t at, char32_t first, size_t width, size_t* skip) const {const Bucket* b = BucketOf(first);if (b == nullptr || b->tokens.empty()) return nullptr;if (!b->whole) {@@ -1215,7 +1272,10 @@ class AddedVocab {}for (const AddedToken* t : b->tokens) {if (text.compare(at, t->content.size(), t->content) != 0) continue;- if (t->single_word && !StandsAlone(text, at, at + t->content.size())) continue;+ if (t->single_word && !StandsAlone(text, at, at + t->content.size())) {+ *skip = t->content.size();+ return nullptr;+ }return t;}return nullptr;@@ -1280,6 +1340,13 @@ class Bpe {Refuse("merge at rank " + std::to_string(rank) + " produces \"" + pair.first + pair.second +"\", which the vocabulary does not contain");}+ // The crate builds byte-fallback and unknown tokens before merging; this port applies them after.+ for (const std::string* name : {&pair.first, &pair.second, &merged->first}) {+ unsigned char b;+ if ((byte_fallback && ByteToken(*name, &b)) || (unk != nullptr && *name == *unk)) {+ Unsupported("merge at rank " + std::to_string(rank) + " names the byte or unknown token \"" + *name + "\"");+ }+ }merges_.emplace(PairKey(l->second, r->second), MergeRule{int32_t(rank), merged->second}); // the lowest rank wins}if (byte_fallback) {@@ -1517,8 +1584,13 @@ std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value*"\". Refusing to load rather than mis-tokenize silently");}auto impl = std::make_shared<Impl>();- impl->normalizer = ParseNormalizer(Member(doc, "normalizer"));+ bool normalizer_may_drop = false;+ impl->normalizer = ParseNormalizer(Member(doc, "normalizer"), &normalizer_may_drop);impl->pre = ParsePreTokenizer(Member(doc, "pre_tokenizer"));+ if (normalizer_may_drop && impl->pre->PrependsFirstOnly()) {+ // The crate decides "first" by the original offset of the normalized text, which this port does not track.+ Unsupported("a Metaspace prepend_scheme \"first\" after a normalizer that can delete leading text (Strip, Replace)");+ }impl->decoder = ParseDecoder(Member(doc, "decoder"));impl->byte_level_only = dynamic_cast<const ByteLevel*>(impl->pre.get()) != nullptr;@@ -1541,6 +1613,12 @@ std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value*Refuse("added token id " + std::to_string(i) + " is claimed by both \"" + clash->second + "\" and \"" + content + "\"");}seen[i] = content;+ for (const char* flag : {"special", "lstrip", "rstrip", "single_word", "normalized"}) {+ const Value* f = r.Find(flag);+ if (f == nullptr || f->kind() != Value::Kind::Bool) {+ Refuse("added token \"" + content + "\" must declare " + flag + " as a boolean");+ }+ }added.push_back(AddedToken{content, i, Bool(r, "special", false), Bool(r, "lstrip", false), Bool(r, "rstrip", false),Bool(r, "single_word", false), Bool(r, "normalized", false)});}@@ -1555,6 +1633,18 @@ std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value*if (v.as_int() < 0) Refuse("the vocabulary holds a negative id " + std::to_string(v.as_int()));vocab[vocab_v->key(i)] = int32_t(v.as_int());}+ {+ int32_t next = int32_t(vocab.size());+ for (auto& tok : added) {+ auto known = vocab.find(tok.content);+ int32_t expect = known != vocab.end() ? known->second : next++;+ if (known == vocab.end()) vocab[tok.content] = expect;+ if (tok.id != expect) {+ Refuse("added token \"" + tok.content + "\" declares id " + std::to_string(tok.id) + ", but the reference gives it " ++ std::to_string(expect) + " (its vocabulary id, or the next free id in file order)");+ }+ }+ }for (const auto& tok : added) {if (tok.special) {impl->is_special.insert(tok.id);@@ -1580,6 +1670,13 @@ std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value*std::vector<std::pair<std::string, std::string>> merges;ReadMerges(model, merges);+ for (const char* key : {"end_of_word_suffix", "continuing_subword_prefix"}) {+ const Value* v = Member(model, key);+ if (v != nullptr && v->kind() == Value::Kind::String && !v->as_string().empty()) {+ Unsupported(std::string("model.") + key + " is not implemented");+ }+ }+ if (Member(model, "dropout") != nullptr) Unsupported("model.dropout is not implemented");std::string unk;const Value* unk_v = Member(model, "unk_token");if (unk_v != nullptr && unk_v->kind() != Value::Kind::String) Refuse("model.unk_token must be a string or null");@@ -1689,9 +1786,9 @@ std::vector<int32_t> Tokenizer::Encode(std::string_view text_in, int64_t limit)std::string text = ud::ToValid(text_in); // a Python str is always validout.reserve(size_t(limit > 0 && limit < int64_t(text.size() / 3 + 16) ? limit : int64_t(text.size() / 3 + 16)));auto full = [&]() { return limit >= 0 && int64_t(out.size()) >= limit; };- auto plain = [&](const std::string& s) {+ auto plain = [&](const std::string& s, bool origin) {std::vector<std::string> pieces;- t.pre->Split(s, pieces);+ t.pre->Split(s, pieces, origin);for (const auto& piece : pieces) {if (full()) return;t.bpe->EncodePiece(piece, out);@@ -1704,14 +1801,15 @@ std::vector<int32_t> Tokenizer::Encode(std::string_view text_in, int64_t limit)continue;}std::string normalized = t.normalizer(sp.text);+ bool origin = sp.offset == 0; // only the text at the start of the input is "first"if (t.normal_added.empty) {- plain(normalized);+ plain(normalized, origin);continue;}for (const Span& inner : t.normal_added.Split(normalized)) {if (full()) break;if (inner.is_token) out.push_back(inner.id);- else plain(inner.text);+ else plain(inner.text, origin && inner.offset == 0);}}if (limit >= 0 && int64_t(out.size()) > limit) out.resize(size_t(limit));@@ -1768,7 +1866,7 @@ std::string Tokenizer::MaskToken() const { return impl_->mask_token; }size_t Tokenizer::CountMergeDisagreements(std::string_view text, size_t* pieces_checked) const {size_t bad = 0;std::vector<std::string> pieces;- impl_->pre->Split(impl_->normalizer(ud::ToValid(text)), pieces);+ impl_->pre->Split(impl_->normalizer(ud::ToValid(text)), pieces, true);for (const auto& p : pieces) {if (!impl_->bpe->StrategiesAgree(p)) bad++;if (pieces_checked != nullptr) (*pieces_checked)++;@@ -1779,7 +1877,7 @@ size_t Tokenizer::CountMergeDisagreements(std::string_view text, size_t* pieces_std::string Tokenizer::NormalizeForTest(std::string_view text) const { return impl_->normalizer(ud::ToValid(text)); }std::vector<std::string> Tokenizer::PreTokenizeForTest(std::string_view normalized) const {std::vector<std::string> out;- impl_->pre->Split(normalized, out);+ impl_->pre->Split(normalized, out, true);return out;}@@ -1801,7 +1899,7 @@ Result<std::vector<std::string>> RunPreTokenizerSpec(std::string_view spec_json,auto v = pyvalue::Parse(spec_json);if (!v.ok()) return Result<std::vector<std::string>>::Fail(v.error());std::vector<std::string> out;- ParsePreTokenizer(&v.value())->Split(input, out);+ ParsePreTokenizer(&v.value())->Split(input, out, true);return Result<std::vector<std::string>>::Ok(out);} catch (const Fail& f) {return Result<std::vector<std::string>>::Fail(f.msg);diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cppindex b496077a..ca59a222 100644--- a/laya-cpp/tests/tokenizer_test.cpp+++ b/laya-cpp/tests/tokenizer_test.cpp@@ -1,6 +1,7 @@// laya::tokenizer against the `tokenizers` crate (through the Python-recorded goldens) and against// hand-built tokenizers. Ported from laya-go/tokenizer's tests.#include <algorithm>+#include <chrono>#include <cstdlib>#include <fstream>#include <mutex>@@ -86,6 +87,17 @@ Tokenizer MustParse(const std::string& json, const char* config = nullptr) {return std::move(t).value();}+// An added_tokens entry with every field the crate requires; `on` names the flags that are true.+std::string Tok(const std::string& content, int id, std::initializer_list<const char*> on = {}) {+ std::string out = "{\"content\":\"" + content + "\",\"id\":" + std::to_string(id);+ for (const char* flag : {"special", "lstrip", "rstrip", "single_word", "normalized"}) {+ bool v = false;+ for (const char* o : on) v = v || std::string(o) == flag;+ out += std::string(",\"") + flag + "\":" + (v ? "true" : "false");+ }+ return out + "}";+}+// A tiny byte-level tokenizer built in the test, so the loader is checked without a checkpoint.std::string ByteLevelSpec(const std::string& extra) {std::string vocab;@@ -181,11 +193,11 @@ TEST(TokenizerNormalize, LowercaseMatchesTheCrateOverAllUnicode) {TEST(TokenizerAdded, SingleWordFlankMatchesTheCrate) {LAYA_GOLDEN_LIST(sw, "go_extra.json", "single_word");Tokenizer tok = MustParse(- R"({"model":{"type":"BPE","vocab":{"a":0,"<unk>":2},"merges":[],"unk_token":"<unk>"},- "added_tokens":[{"content":"TOK","id":1,"single_word":true,"special":true}]})");+ R"({"model":{"type":"BPE","vocab":{"a":0,"<unk>":1},"merges":[],"unk_token":"<unk>"},+ "added_tokens":[{"content":"TOK","id":2,"single_word":true,"special":true,"lstrip":false,"rstrip":false,"normalized":false}]})");auto standalone = [&](const std::string& s) {for (int32_t id : tok.Encode(s)) {- if (id == 1) return std::string("1");+ if (id == 2) return std::string("1");}return std::string("0");};@@ -478,7 +490,7 @@ TEST(TokenizerLoader, Refuses) {{"a merge that is not a pair", Replace(good, "\"a b\"", "[\"a\"]"), false},{"a space-joined merge with an empty half", Replace(good, "\"a b\"", "\"a \""), false},{"an added token with no id", ByteLevelSpec(",\"added_tokens\":[{\"content\":\"x\"}]"), false},- {"two added tokens sharing an id", ByteLevelSpec(",\"added_tokens\":[{\"content\":\"x\",\"id\":300},{\"content\":\"y\",\"id\":300}]"), false},+ {"two added tokens sharing an id", ByteLevelSpec(",\"added_tokens\":[" + Tok("x", 257) + "," + Tok("y", 257) + "]"), false},{"a vocabulary giving one id to two tokens", Replace(good, "\"ab\":256", "\"ab\":0"), false},{"no vocabulary", "{\"model\":{\"type\":\"BPE\",\"merges\":[]}}", false},{"not JSON", "{", false},@@ -520,26 +532,24 @@ TEST(TokenizerLoader, IdsFarBeyondTheVocabularyAreRefused) {TEST(TokenizerAdded, LongestLstripRstripSingleWordNormalized) {Tokenizer tok = MustParse(ByteLevelSpec(- ",\"added_tokens\":[{\"content\":\"<S>\",\"id\":300,\"special\":true,\"normalized\":false},"- "{\"content\":\"<SS>\",\"id\":301,\"special\":true,\"normalized\":false},"- "{\"content\":\"[M]\",\"id\":302,\"special\":true,\"lstrip\":true,\"rstrip\":true},"- "{\"content\":\"word\",\"id\":303,\"single_word\":true},"- "{\"content\":\"\xC3\xA9" "clair\",\"id\":304,\"normalized\":true},"- "{\"content\":\"\xC2\xA7\",\"id\":305},{\"content\":\"!\",\"id\":306}],\"decoder\":{\"type\":\"ByteLevel\"}"));+ ",\"added_tokens\":[" + Tok("<S>", 257, {"special"}) + "," + Tok("<SS>", 258, {"special"}) + "," ++ Tok("[M]", 259, {"special", "lstrip", "rstrip"}) + "," + Tok("word", 260, {"single_word"}) + "," ++ Tok("\xC3\xA9" "clair", 261, {"normalized"}) + "," + Tok("\xC2\xA7", 167) + "," + Tok("!", 33) ++ "],\"decoder\":{\"type\":\"ByteLevel\"}"));using V = std::vector<int32_t>;- EXPECT_EQ(tok.Encode("<SS><S>"), (V{301, 300})) << "the longest token wins at a position";- EXPECT_EQ(tok.Encode("a [M] b"), (V{97, 302, 98})) << "lstrip and rstrip absorb the white space";- EXPECT_EQ(tok.Encode("word"), (V{303}));+ EXPECT_EQ(tok.Encode("<SS><S>"), (V{258, 257})) << "the longest token wins at a position";+ EXPECT_EQ(tok.Encode("a [M] b"), (V{97, 259, 98})) << "lstrip and rstrip absorb the white space";+ EXPECT_EQ(tok.Encode("word"), (V{260}));auto inside = tok.Encode("swords word.");ASSERT_GE(inside.size(), 3u);- EXPECT_EQ(inside[inside.size() - 2], 303) << "a single_word token inside a word is not matched but a free one is";- EXPECT_EQ(std::count(inside.begin(), inside.end(), 303), 1) << "swords holds the word but not as a word";- EXPECT_EQ(tok.Encode("\xC3\xA9" "clair"), (V{304}));- EXPECT_EQ(tok.Encode("a\xC2\xA7" "b!"), (V{97, 305, 98, 306}));+ EXPECT_EQ(inside[inside.size() - 2], 260) << "a single_word token inside a word is not matched but a free one is";+ EXPECT_EQ(std::count(inside.begin(), inside.end(), 260), 1) << "swords holds the word but not as a word";+ EXPECT_EQ(tok.Encode("\xC3\xA9" "clair"), (V{261}));+ EXPECT_EQ(tok.Encode("a\xC2\xA7" "b!"), (V{97, 167, 98, 33})) << "added tokens already in the vocabulary keep its ids";EXPECT_EQ(tok.Encode("<S"), (V{60, 83})) << "a prefix of a token is not the token";- EXPECT_EQ(tok.Decode({300, 97, 301}, true), "a");- EXPECT_EQ(tok.Decode({300, 97}, false), "<S>a");- EXPECT_EQ(tok.SpecialId("<S>"), std::optional<int32_t>(300));+ EXPECT_EQ(tok.Decode({257, 97, 258}, true), "a");+ EXPECT_EQ(tok.Decode({257, 97}, false), "<S>a");+ EXPECT_EQ(tok.SpecialId("<S>"), std::optional<int32_t>(257));EXPECT_FALSE(tok.SpecialId("word").has_value());EXPECT_EQ(tok.SpecialTokens().size(), 3u);}@@ -551,13 +561,13 @@ TEST(TokenizerLoader, RolesFromTokenizerConfigAndDirectoryLayout) {};std::string cmd = "mkdir -p '" + base + "/m/tokenizer' '" + base + "/plain' '" + base + "/empty'";ASSERT_EQ(std::system(cmd.c_str()), 0);- std::string spec = ByteLevelSpec(",\"added_tokens\":[{\"content\":\"<c>\",\"id\":300,\"special\":true},{\"content\":\"<m>\",\"id\":301,\"special\":true}]");+ std::string spec = ByteLevelSpec(",\"added_tokens\":[" + Tok("<c>", 257, {"special"}) + "," + Tok("<m>", 258, {"special"}) + "]");write(base + "/plain/tokenizer.json", spec);write(base + "/plain/tokenizer_config.json", "{\"cls_token\":\"<c>\",\"mask_token\":{\"content\":\"<m>\"},\"pad_token\":null}");auto t = Tokenizer::LoadDir(base + "/plain");ASSERT_TRUE(t.ok()) << t.error();- EXPECT_EQ(t.value().Cls(), std::optional<int32_t>(300));- EXPECT_EQ(t.value().Mask(), std::optional<int32_t>(301));+ EXPECT_EQ(t.value().Cls(), std::optional<int32_t>(257));+ EXPECT_EQ(t.value().Mask(), std::optional<int32_t>(258));EXPECT_EQ(t.value().MaskToken(), "<m>");EXPECT_FALSE(t.value().Pad().has_value()) << "a null role is absent";EXPECT_FALSE(t.value().Sep().has_value()) << "an unnamed role is absent";@@ -696,8 +706,8 @@ TEST(TokenizerRefusals, SequenceRunsInOrder) {}TEST(TokenizerAdded, RstripAbsorbsAllWhiteSpace) {- Tokenizer tok = MustParse(ByteLevelSpec(",\"added_tokens\":[{\"content\":\"[M]\",\"id\":302,\"special\":true,\"rstrip\":true}]"));- EXPECT_EQ(tok.Encode("[M] \t b"), (std::vector<int32_t>{302, 98}));+ Tokenizer tok = MustParse(ByteLevelSpec(",\"added_tokens\":[" + Tok("[M]", 257, {"special", "rstrip"}) + "]"));+ EXPECT_EQ(tok.Encode("[M] \t b"), (std::vector<int32_t>{257, 98}));}// Canonical ordering must run before composition: a + dot above (230) + dot below (220) is a-dot-below@@ -707,3 +717,262 @@ TEST(TokenizerNormalize, MarksAreReorderedBeforeTheyCompose) {EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFKC), "\xE1\xBA\xA1\xCC\x87");EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFD), "a\xCC\xA3\xCC\x87");}++// ---- review round 1: every case's expected value is the crate's (tokenizers 0.23.2) ---------------------+namespace {++std::string NormSpec(const std::string& pattern, const std::string& to = "_") {+ return "{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"" + pattern + "\"},\"content\":\"" + to + "\"}";+}++// Letters a-z are 0-25, the marker is 26, <unk> is 27 and the first added token is 28.+std::string MetaspaceSpec(const std::string& pre, const std::string& normalizer = "null") {+ std::string vocab;+ for (char c = 'a'; c <= 'z'; c++) vocab += std::string("\"") + c + "\":" + std::to_string(c - 'a') + ",";+ vocab += "\"\xE2\x96\x81\":26,\"<unk>\":27";+ return "{\"model\":{\"type\":\"BPE\",\"vocab\":{" + vocab + "},\"merges\":[],\"unk_token\":\"<unk>\"},\"normalizer\":" + normalizer ++ ",\"pre_tokenizer\":" + pre + ",\"added_tokens\":[" + Tok("<m>", 28, {"special"}) + "]}";+}++std::string Metaspace(const char* scheme, bool split = true) {+ return std::string("{\"type\":\"Metaspace\",\"replacement\":\"\xE2\x96\x81\",\"prepend_scheme\":\"") + scheme + "\",\"split\":" ++ (split ? "true" : "false") + "}";+}++using Ids32 = std::vector<int32_t>;++} // namespace++TEST(TokenizerRegex, RepeatsAreBoundedByProgramSizeAndDepth) {+ // (((a{1000}){1000}){10})x built about 450 MB of program before the cap.+ for (const char* p : {"(((a{1000}){1000}){30})x", "((a{300}){300}){2}"}) {+ auto t0 = std::chrono::steady_clock::now();+ auto r = tk::RunNormalizerSpec(NormSpec(p), "aaa");+ auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0).count();+ ASSERT_FALSE(r.ok()) << p;+ EXPECT_FALSE(tk::IsUnsupported(r.error())) << r.error();+ EXPECT_LT(ms, 600) << p << " was expanded before it was refused";+ }+ EXPECT_TRUE(tk::RunNormalizerSpec(NormSpec("a{1000}b{1000}"), "ab").ok()) << "a large but ordinary repeat still loads";+ auto deep = [](size_t n) { return std::string(n, '(') + "a" + std::string(n, ')'); };+ EXPECT_EQ(tk::RunNormalizerSpec(NormSpec(deep(100)), "baab").value(), "b__b") << "the crate gives b__b";+ EXPECT_TRUE(tk::RunNormalizerSpec(NormSpec(deep(256)), "a").ok());+ EXPECT_FALSE(tk::RunNormalizerSpec(NormSpec(deep(257)), "a").ok());+ EXPECT_FALSE(tk::RunNormalizerSpec(NormSpec(deep(200000)), "a").ok()) << "was a stack overflow";+ std::string siblings;+ for (int i = 0; i < 300; i++) siblings += "(a)";+ EXPECT_EQ(tk::RunNormalizerSpec(NormSpec(siblings), "a").value(), "a") << "groups side by side are not nested";+ std::string wide;+ for (int i = 0; i < 60000; i++) wide += "a|";+ EXPECT_FALSE(tk::RunNormalizerSpec(NormSpec(wide + "b"), "a").ok()) << "120000 instructions";+}++TEST(TokenizerRegex, ReplaceAllIsLinearInTheText) {+ std::string text(400000, 'a');+ auto t0 = std::chrono::steady_clock::now();+ auto r = tk::RunNormalizerSpec(NormSpec("a"), text);+ auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0).count();+ ASSERT_TRUE(r.ok());+ EXPECT_EQ(r.value(), std::string(400000, '_'));+ EXPECT_LT(ms, 3000) << "a table sized by the whole text was allocated per match";+}++TEST(TokenizerRegex, SparseTableAgreesWithTheCrate) {+ // A program big enough that (instruction x position) exceeds the dense table uses the sparse one.+ std::string big(100000, 'b');+ auto s = tk::RunNormalizerSpec(NormSpec("(x{1000}y{1000}z{1000}|b)"), big + "x");+ ASSERT_TRUE(s.ok()) << s.error();+ EXPECT_EQ(s.value(), std::string(100000, '_') + "x");+ // Without the memo this is exponential, and it must also hold on the sparse table.+ auto fib = tk::RunNormalizerSpec(NormSpec("(a|aa)+b|x{1000}y{1000}z{1000}"), std::string(5000, 'a') + std::string(95000, 'c') + "x");+ ASSERT_TRUE(fib.ok()) << fib.error();+ EXPECT_EQ(fib.value(), std::string(5000, 'a') + std::string(95000, 'c') + "x");+ // Matches after the first share the table; each must still be found.+ EXPECT_EQ(tk::RunNormalizerSpec(NormSpec("a+?b?|b"), "aab ab b").value(), "__ _ _");+}++TEST(TokenizerRegex, CaseInsensitiveMatchesTheCrateOrIsRefused) {+ auto run = [](const std::string& pattern, const std::string& in) {+ auto r = tk::RunNormalizerSpec(NormSpec(pattern), in);+ EXPECT_TRUE(r.ok()) << pattern << ": " << r.error();+ return r.ok() ? r.value() : std::string("<error>");+ };+ EXPECT_EQ(run("(?i)[a-c]+", "AbCd"), "_d");+ EXPECT_EQ(run("(?i)[^a-c]", "aBcD"), "aBc_");+ EXPECT_EQ(run("(?i)[A-C]", "abcABC"), "______");+ EXPECT_EQ(run("(?i)x[a-c]y", "XAY xcY xdy"), "_ _ xdy");+ EXPECT_EQ(run("(?i)q|Z", "qQzZ"), "____");+ // The crate folds U+212A (Kelvin) to k, U+017F and U+00DF to s and ss, and ligatures to letter pairs.+ for (const char* p : {"(?i)k", "(?i)K", "(?i)ss", "(?i)[a-z]", "(?i)[^k]", "(?i)f", "(?i)fi", "(?i)(?:s)t", "(?i)[k-k]"}) {+ auto r = tk::RunNormalizerSpec(NormSpec(p), "a");+ ASSERT_FALSE(r.ok()) << p;+ EXPECT_TRUE(tk::IsUnsupported(r.error())) << p << ": " << r.error();+ }+ EXPECT_EQ(run("k", "kK"), "_K") << "case-sensitive k is not affected";+ EXPECT_EQ(run("ss", "ssSs"), "_Ss");+}++TEST(TokenizerPre, MetaspaceFirstPrependsOnlyAtTheStartOfTheInput) {+ struct Row {+ const char* scheme;+ const char* text;+ Ids32 want;+ } rows[] = {+ {"first", "a<m>b c", {26, 0, 28, 1, 26, 2}}, {"first", "<m>b c", {28, 1, 26, 2}},+ {"first", "b", {26, 1}}, {"first", " <m>b", {26, 28, 1}},+ {"first", "<m><m>b", {28, 28, 1}}, {"first", "<m>b<m>c", {28, 1, 28, 2}},+ {"first", "a<m>b<m>c", {26, 0, 28, 1, 28, 2}}, {"first", "a b<m> c", {26, 0, 26, 1, 28, 26, 2}},+ {"always", "a<m>b c", {26, 0, 28, 26, 1, 26, 2}}, {"always", "<m>b c", {28, 26, 1, 26, 2}},+ {"never", "a<m>b c", {0, 28, 1, 26, 2}}, {"never", "b", {1}},+ };+ for (const auto& row : rows) {+ Tokenizer tok = MustParse(MetaspaceSpec(Metaspace(row.scheme)));+ EXPECT_EQ(tok.Encode(row.text), row.want) << row.scheme << " " << row.text;+ }+ // Only the first piece of an earlier split is still at the start.+ Tokenizer bytes = MustParse(MetaspaceSpec("{\"type\":\"Sequence\",\"pretokenizers\":[{\"type\":\"ByteLevel\",\"add_prefix_space\":false}," ++ Metaspace("first") + "]}"));+ EXPECT_EQ(bytes.Encode("ab cd"), (Ids32{26, 0, 1, 27, 2, 3}));+ EXPECT_EQ(bytes.Encode("ab12 cd"), (Ids32{26, 0, 1, 27, 27, 27, 2, 3}));+ Tokenizer seq = MustParse(MetaspaceSpec("{\"type\":\"Sequence\",\"pretokenizers\":[" + Metaspace("first") + "," + Metaspace("first") + "]}"));+ EXPECT_EQ(seq.Encode("ab<m>ab"), (Ids32{26, 0, 1, 28, 0, 1}));+ Tokenizer nosplit = MustParse(MetaspaceSpec(Metaspace("first", false)));+ EXPECT_EQ(nosplit.Encode("a b<m>c d"), (Ids32{26, 0, 26, 1, 28, 2, 26, 3}));+ // The normalizer runs on each piece; the second piece is still not at the start.+ Tokenizer prepend = MustParse(MetaspaceSpec(Metaspace("first"), R"({"type":"Prepend","prepend":"x"})"));+ EXPECT_EQ(prepend.Encode("a<m>b"), (Ids32{26, 23, 0, 28, 23, 1}));+ // The crate keys "first" on the original offset, which a normalizer that deletes text changes.+ auto strip = Tokenizer::Parse(MetaspaceSpec(Metaspace("first"), R"({"type":"Strip","strip_left":true,"strip_right":false})"));+ ASSERT_FALSE(strip.ok());+ EXPECT_TRUE(tk::IsUnsupported(strip.error())) << strip.error();+ EXPECT_TRUE(Tokenizer::Parse(MetaspaceSpec(Metaspace("always"), R"({"type":"Strip","strip_left":true,"strip_right":false})")).ok());+ auto replace = Tokenizer::Parse(MetaspaceSpec(Metaspace("first"), R"({"type":"Replace","pattern":{"String":"z"},"content":""})"));+ ASSERT_FALSE(replace.ok());+ EXPECT_TRUE(tk::IsUnsupported(replace.error())) << replace.error();+ EXPECT_EQ(MustParse(MetaspaceSpec(Metaspace("first"), R"({"type":"Strip","strip_left":false,"strip_right":true})")).Encode("ab "),+ (Ids32{26, 0, 1})) << "a right strip leaves the start alone";+ // An added token matched in the normalized text: its pieces are still not at the start.+ std::string normalized_token = MetaspaceSpec(Metaspace("first"));+ normalized_token.replace(normalized_token.find("\"normalized\":false"), 18, "\"normalized\":true");+ Tokenizer inner = MustParse(normalized_token);+ EXPECT_EQ(inner.Encode("a<m>b c"), (Ids32{26, 0, 28, 1, 26, 2}));+ EXPECT_EQ(inner.Encode("<m>b c"), (Ids32{28, 1, 26, 2}));+ EXPECT_EQ(inner.Encode(" <m>b"), (Ids32{26, 28, 1}));+}++TEST(TokenizerPre, MetaspaceLegacyFlagMustAgreeWithTheScheme) {+ auto spec = [](const std::string& tail) {+ return MetaspaceSpec("{\"type\":\"Metaspace\",\"replacement\":\"\xE2\x96\x81\"" + tail + "}");+ };+ // The crate: false with any scheme but "never" (or none) is an error; true with "never" is accepted.+ for (const char* tail : {",\"add_prefix_space\":false", ",\"add_prefix_space\":false,\"prepend_scheme\":\"always\"",+ ",\"add_prefix_space\":false,\"prepend_scheme\":\"first\""}) {+ EXPECT_FALSE(Tokenizer::Parse(spec(tail)).ok()) << tail;+ }+ EXPECT_EQ(MustParse(spec(",\"add_prefix_space\":false,\"prepend_scheme\":\"never\"")).Encode("ab"), (Ids32{0, 1}));+ EXPECT_EQ(MustParse(spec(",\"add_prefix_space\":true,\"prepend_scheme\":\"never\"")).Encode("ab"), (Ids32{0, 1}));+ EXPECT_EQ(MustParse(spec(",\"add_prefix_space\":true")).Encode("ab"), (Ids32{26, 0, 1}));+ EXPECT_EQ(MustParse(spec("")).Encode("ab"), (Ids32{26, 0, 1}));+}++TEST(TokenizerAdded, SingleWordFailureSkipsTheWholeLongestHit) {+ // xyz is single_word, xy is free. The crate drops a longest hit that fails the flank test and resumes+ // after it, so xy inside "wxyz" is not found.+ Tokenizer tok = MustParse(ByteLevelSpec(",\"added_tokens\":[" + Tok("xyz", 257, {"single_word"}) + "," + Tok("xy", 258) + "," + Tok("yz", 259) + "]"));+ struct Row {+ const char* text;+ Ids32 want;+ } rows[] = {+ {"wxyz", {119, 120, 121, 122}},+ {"xyz", {257}},+ {"xyzw xyz", {120, 121, 122, 119, 32, 257}},+ {"xyzw xy", {120, 121, 122, 119, 32, 258}},+ {"w xyz w", {119, 32, 257, 32, 119}},+ {"xyz,wxyz", {257, 44, 119, 120, 121, 122}},+ {"wxyz yz", {119, 120, 121, 122, 32, 259}}, // yz inside the failed xyz is not found either+ {"xy", {258}},+ {"wxy", {119, 258}},+ };+ for (const auto& row : rows) EXPECT_EQ(tok.Encode(row.text), row.want) << row.text;+}++TEST(TokenizerLoader, AddedTokenIdsFollowTheCrate) {+ // The crate keeps the vocabulary id of an added token already in the vocabulary and numbers the others+ // from the vocabulary size in file order; the id in the file is not used. Ids that differ are refused.+ auto load = [](const std::string& tokens) { return Tokenizer::Parse(ByteLevelSpec(",\"added_tokens\":[" + tokens + "]")); };+ EXPECT_FALSE(load(Tok("ab", 300, {"special"})).ok()) << "ab is vocabulary id 256 in the crate";+ EXPECT_TRUE(load(Tok("ab", 256, {"special"})).ok());+ EXPECT_FALSE(load(Tok("<x>", 300)).ok()) << "the crate numbers it 257";+ EXPECT_FALSE(load(Tok("<x>", 257) + "," + Tok("<y>", 259)).ok()) << "the crate numbers y 258";+ EXPECT_TRUE(load(Tok("<x>", 257) + "," + Tok("ab", 256) + "," + Tok("<y>", 258)).ok()) << "a vocabulary token takes no new id";+ auto r = load(Tok("<x>", 300));+ EXPECT_NE(r.error().find("257"), std::string::npos) << r.error();+ // The crate needs all five flags as booleans.+ for (const char* flag : {"special", "lstrip", "rstrip", "single_word", "normalized"}) {+ std::string entry = Tok("<x>", 257);+ std::string drop = std::string(",\"") + flag + "\":false";+ entry.erase(entry.find(drop), drop.size());+ EXPECT_FALSE(load(entry).ok()) << "missing " << flag;+ std::string null = Tok("<x>", 257);+ null.replace(null.find(drop), drop.size(), std::string(",\"") + flag + "\":null");+ EXPECT_FALSE(load(null).ok()) << "null " << flag;+ }+}++TEST(TokenizerLoader, ModelOptionsThePortIgnoresAreRefused) {+ std::string good = ByteLevelSpec("");+ auto with = [&](const std::string& field) { return Replace(good, "\"merges\":[\"a b\"]", "\"merges\":[\"a b\"]," + field); };+ for (const char* f : {"\"end_of_word_suffix\":\"</w>\"", "\"continuing_subword_prefix\":\"##\"", "\"dropout\":0.5", "\"dropout\":0.0"}) {+ auto r = Tokenizer::Parse(with(f));+ ASSERT_FALSE(r.ok()) << f;+ EXPECT_TRUE(tk::IsUnsupported(r.error())) << f;+ }+ for (const char* f : {"\"end_of_word_suffix\":\"\"", "\"continuing_subword_prefix\":\"\"", "\"dropout\":null",+ "\"end_of_word_suffix\":null", "\"continuing_subword_prefix\":null"}) {+ EXPECT_TRUE(Tokenizer::Parse(with(f)).ok()) << f << " is the same as absent in the crate";+ }+}++TEST(TokenizerLoader, MergesNamingByteOrUnknownTokensAreRefused) {+ // The crate builds byte-fallback and unknown tokens before merging; this port applies them after.+ auto spec = [](const char* merge, bool byte_fallback, bool unk) {+ return std::string("{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"b\":1,\"ab\":2,\"<0x61>\":3,\"<unk>\":4,\"<0x61>a\":5,\"a<unk>\":6},") ++ "\"merges\":[\"a b\"," + merge + "],\"byte_fallback\":" + (byte_fallback ? "true" : "false") ++ (unk ? ",\"unk_token\":\"<unk>\"" : "") + "}}";+ };+ auto byte_merge = spec("\"<0x61> a\"", true, false);+ auto r = Tokenizer::Parse(byte_merge);+ ASSERT_FALSE(r.ok());+ EXPECT_TRUE(tk::IsUnsupported(r.error())) << r.error();+ auto unk_merge = Tokenizer::Parse(spec("\"a <unk>\"", false, true));+ ASSERT_FALSE(unk_merge.ok());+ EXPECT_TRUE(tk::IsUnsupported(unk_merge.error())) << unk_merge.error();+ EXPECT_TRUE(Tokenizer::Parse(spec("\"<0x61> a\"", false, true)).ok()) << "a byte-shaped name is ordinary without byte_fallback";+ EXPECT_TRUE(Tokenizer::Parse(spec("\"a <unk>\"", true, false)).ok()) << "<unk> is ordinary when it is not the unknown token";+ EXPECT_TRUE(Tokenizer::Parse(spec("\"a b\"", true, true)).ok());+ // A merge that produces a byte token is just as unsound.+ auto produces = Tokenizer::Parse(+ "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"<0x6\":1,\"1>\":2,\"<0x61>\":3},\"merges\":[\"<0x6 1>\"],\"byte_fallback\":true}}");+ ASSERT_FALSE(produces.ok());+ EXPECT_TRUE(tk::IsUnsupported(produces.error())) << produces.error();+}++// Three or more branches left stale jump targets (so "a|a|a|b" matched the empty string), and each `|`+// inserted into the program, which made a long alternation quadratic.+TEST(TokenizerRegex, ManyBranchAlternation) {+ auto run = [](const std::string& pattern, const std::string& in) {+ auto r = tk::RunNormalizerSpec(NormSpec(pattern), in);+ EXPECT_TRUE(r.ok()) << pattern << ": " << r.error();+ return r.ok() ? r.value() : std::string("<error>");+ };+ EXPECT_EQ(run("a|bc|d|e", "abcdexbd"), "____xb_");+ EXPECT_EQ(run("(a|b|c)+d", "abcd bd cad"), "_ _ _");+ EXPECT_EQ(run("x(a|b|c|d)y|z", "xcy xey z"), "_ xey _");+ EXPECT_EQ(run("a|a|a|b", "cab"), "c__");+ std::string wide;+ for (int i = 0; i < 20000; i++) wide += "a|";+ auto t0 = std::chrono::steady_clock::now();+ EXPECT_EQ(run(wide + "b", "cab"), "c__");+ auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0).count();+ EXPECT_LT(ms, 3000);+}