Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(cpp): tokenizer review round 1

Adeshina Lasisif0eadf680e4f
diff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cpp
index 94a00c3a..01a1e06e 100644
--- a/laya-cpp/src/tokenizer.cpp
+++ b/laya-cpp/src/tokenizer.cpp
@@ -282,9 +282,10 @@ class Regex {
std::string ReplaceAll(const std::string& text, const std::string& content) const {
std::string out;
size_t copied = 0, from = 0;
+ Memo memo(prog_.size(), text.size() + 1);
while (from <= text.size()) {
size_t ms, me;
- if (!Find(text, from, &ms, &me)) break;
+ if (!Find(text, from, &ms, &me, memo)) break;
out.append(text, copied, ms - copied);
out += content;
copied = me;
@@ -295,7 +296,8 @@ class Regex {
}
bool MatchesEmpty() const {
size_t s, e;
- return Find("", 0, &s, &e);
+ Memo memo(prog_.size(), 1);
+ return Find("", 0, &s, &e, memo);
}
private:
@@ -312,6 +314,36 @@ class Regex {
std::vector<Inst> prog_;
std::vector<Set> sets_;
+ static constexpr size_t kMaxProgram = 100000; // instructions after repeats are expanded
+ static constexpr int kMaxDepth = 256; // nested groups
+ static constexpr size_t kDenseBits = size_t(1) << 28;
+ int depth_ = 0;
+
+ // The (instruction, position) states already tried. Dense while the table is small, a hash set above
+ // that. One table serves a whole ReplaceAll: a state that failed still fails for any later start, and
+ // the states of a match lie at or before its end, which no later (non-empty) match revisits.
+ class Memo {
+ public:
+ Memo(size_t insts, size_t positions) : insts_(insts), dense_(insts * positions <= kDenseBits) {
+ if (dense_) bits_.assign((insts * positions + 63) / 64, 0);
+ }
+ bool Visit(int pc, size_t pos) { // true the first time
+ size_t key = pos * insts_ + size_t(pc);
+ if (!dense_) return sparse_.insert(key).second;
+ uint64_t& w = bits_[key / 64];
+ uint64_t bit = uint64_t(1) << (key % 64);
+ if (w & bit) return false;
+ w |= bit;
+ return true;
+ }
+
+ private:
+ size_t insts_;
+ bool dense_;
+ std::vector<uint64_t> bits_;
+ std::unordered_set<size_t> sparse_;
+ };
+
// --- parser: pattern -> instructions, emitted directly from a recursive descent -------------
std::vector<char32_t> p_;
size_t at_ = 0;
@@ -320,6 +352,7 @@ class Regex {
bool More() const { return at_ < p_.size(); }
char32_t Peek() const { return p_[at_]; }
int Emit(Op op, char32_t c = 0, int x = 0, int y = 0) {
+ if (prog_.size() >= kMaxProgram) Refuse("a Replace regex that compiles to more than 100000 instructions");
prog_.push_back(Inst{op, c, x, y});
return int(prog_.size()) - 1;
}
@@ -341,34 +374,24 @@ class Regex {
}
}
+ // Each branch starts with a placeholder split (a plain jump for the last one), so no instruction is
+ // ever inserted and every target stays valid.
void Alternation() {
- int start = int(prog_.size());
- Concatenation();
std::vector<int> jumps;
- while (More() && Peek() == '|') {
- at_++;
- // split(start of the left branches, next), the left branch jumps over the right.
- prog_.insert(prog_.begin() + start, Inst{kSplit, 0, 0, 0});
- Shift(start, 1);
- int j = Emit(kJmp);
- jumps.push_back(j);
- prog_[size_t(start)].x = start + 1;
- prog_[size_t(start)].y = int(prog_.size());
+ while (true) {
+ int split = Emit(kSplit);
Concatenation();
- }
- for (int j : jumps) prog_[size_t(j)].x = int(prog_.size());
- }
-
- // Inserting an instruction at `from` moves everything after it, so targets at or past `from`
- // that were emitted before the insertion move by `by`.
- void Shift(int from, int by) {
- for (size_t i = size_t(from) + 1; i < prog_.size(); i++) { // [from] is the inserted instruction
- Inst& in = prog_[i];
- if (in.op == kSplit || in.op == kJmp) {
- if (in.x >= from) in.x += by;
- if (in.op == kSplit && in.y >= from) in.y += by;
+ if (More() && Peek() == '|') {
+ at_++;
+ jumps.push_back(Emit(kJmp)); // the finished branch jumps over the rest
+ prog_[size_t(split)].x = split + 1;
+ prog_[size_t(split)].y = int(prog_.size());
+ continue;
}
+ prog_[size_t(split)] = Inst{kJmp, 0, split + 1, 0};
+ break;
}
+ for (int j : jumps) prog_[size_t(j)].x = int(prog_.size());
}
void Concatenation() {
@@ -406,6 +429,7 @@ class Regex {
prog_.resize(size_t(start));
if (lo > 1000 || hi > 1000) Refuse("a Replace regex repeat count above 1000");
auto copy = [&]() {
+ if (prog_.size() + body.size() > kMaxProgram) Refuse("a Replace regex that compiles to more than 100000 instructions");
int base = int(prog_.size());
for (Inst in : body) {
if (in.op == kSplit || in.op == kJmp) {
@@ -492,7 +516,9 @@ class Regex {
if (at_ + 1 < p_.size() && p_[at_ + 1] == ':') at_ += 2;
else Unsupported("this Replace regex group syntax (?...) is not implemented");
}
+ if (++depth_ > kMaxDepth) Refuse("a Replace regex nested more than 256 groups deep");
Alternation();
+ depth_--;
if (!More() || Peek() != ')') Refuse("a Replace normalizer's regex: missing )");
at_++;
} else if (c == '[') {
@@ -533,7 +559,17 @@ class Regex {
Unsupported("the Replace regex escape \\" + std::string(1, char(c < 0x80 ? c : '?')) + " is not implemented");
}
+ // Oniguruma folds with full Unicode rules: K (Kelvin) is k, the long s and the sharp s are s/ss, and
+ // the ligatures ff, fi, fl, ffi, ffl, st fold to letter pairs. An ASCII-only fold would differ.
+ void FoldTrap(char32_t lo, char32_t hi) const {
+ if (!icase_) return;
+ for (char32_t t : {U'k', U'K', U's', U'S', U'f', U'F'}) {
+ if (lo <= t && t <= hi) Unsupported("a case-insensitive Replace regex with k, s or f, whose Unicode case folding is not implemented");
+ }
+ }
+
void Literal(char32_t c) {
+ FoldTrap(c, c);
if (icase_ && ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) {
Set s;
s.ranges = {{c | 0x20, c | 0x20}, {c & ~char32_t(0x20), c & ~char32_t(0x20)}};
@@ -567,6 +603,7 @@ class Regex {
hi = h == '\\' ? Escape() : (at_++, h);
if (hi < lo) Refuse("a Replace normalizer's regex: a range with its ends reversed");
}
+ FoldTrap(lo, hi);
s.ranges.push_back({lo, hi});
}
if (icase_) {
@@ -585,9 +622,7 @@ class Regex {
}
// --- matcher --------------------------------------------------------------------------------
- bool Find(const std::string& text, size_t from, size_t* ms, size_t* me) const {
- size_t width = text.size() + 1;
- std::vector<bool> seen(prog_.size() * width, false);
+ bool Find(const std::string& text, size_t from, size_t* ms, size_t* me, Memo& seen) const {
std::vector<std::pair<int, size_t>> stack;
for (size_t start = from; start <= text.size();) {
stack.assign(1, {0, start});
@@ -595,9 +630,7 @@ class Regex {
auto [pc, pos] = stack.back();
stack.pop_back();
while (true) {
- size_t key = size_t(pc) * width + pos;
- if (seen[key]) break;
- seen[key] = true;
+ if (!seen.Visit(pc, pos)) break;
const Inst& in = prog_[size_t(pc)];
if (in.op == kMatch) {
*ms = start;
@@ -707,7 +740,7 @@ std::string ReplaceAllLiteral(const std::string& s, const std::string& from, con
return out;
}
-Normalizer ParseNormalizer(const Value* n) {
+Normalizer ParseNormalizer(const Value* n, bool* may_drop = nullptr) {
if (n == nullptr || n->is_null()) return [](const std::string& s) { return s; };
std::string kind = TypeOf(*n, "normalizer");
if (kind == "NFC") return [](const std::string& s) { return NormalizeImpl(s, Form::NFC); };
@@ -721,9 +754,11 @@ Normalizer ParseNormalizer(const Value* n) {
}
if (kind == "Strip") {
bool l = Bool(*n, "strip_left", true), r = Bool(*n, "strip_right", true);
+ if (l && may_drop != nullptr) *may_drop = true;
return [l, r](const std::string& s) { return StripImpl(s, l, r); };
}
if (kind == "Replace") {
+ if (may_drop != nullptr) *may_drop = true;
const Value& pattern = Object(*n, "pattern");
const Value* content = n->Find("content");
if (content == nullptr || content->kind() != Value::Kind::String) Refuse("a Replace normalizer must declare content");
@@ -750,7 +785,7 @@ Normalizer ParseNormalizer(const Value* n) {
std::vector<Normalizer> steps;
for (const Value& child : Array(*n, "normalizers").items()) {
if (child.kind() != Value::Kind::Object) Refuse("a normalizer in a Sequence must be an object");
- steps.push_back(ParseNormalizer(&child));
+ steps.push_back(ParseNormalizer(&child, may_drop));
}
return [steps](const std::string& s) {
std::string cur = s;
@@ -792,12 +827,14 @@ const Alphabet& TheAlphabet() {
class PreTokenizer {
public:
virtual ~PreTokenizer() = default;
- virtual void Split(std::string_view text, std::vector<std::string>& out) const = 0;
+ // `origin`: the text starts where the input did (not after an added token).
+ virtual void Split(std::string_view text, std::vector<std::string>& out, bool origin) const = 0;
+ virtual bool PrependsFirstOnly() const { return false; }
};
class WholeText : public PreTokenizer {
public:
- void Split(std::string_view text, std::vector<std::string>& out) const override {
+ void Split(std::string_view text, std::vector<std::string>& out, bool) const override {
if (!text.empty()) out.emplace_back(text);
}
};
@@ -805,7 +842,7 @@ class WholeText : public PreTokenizer {
class ByteLevel : public PreTokenizer {
public:
ByteLevel(bool prefix, bool regex) : add_prefix_space_(prefix), use_regex_(regex) {}
- void Split(std::string_view text, std::vector<std::string>& out) const override {
+ void Split(std::string_view text, std::vector<std::string>& out, bool) const override {
if (text.empty()) return;
std::string source(text);
if (add_prefix_space_ && source[0] != ' ') source = " " + source;
@@ -874,12 +911,13 @@ class ByteLevel : public PreTokenizer {
class Metaspace : public PreTokenizer {
public:
- Metaspace(std::string replacement, bool prepend, bool split)
- : replacement_(std::move(replacement)), prepend_(prepend), split_(split) {}
- void Split(std::string_view text, std::vector<std::string>& out) const override {
+ Metaspace(std::string replacement, bool prepend, bool first_only, bool split)
+ : replacement_(std::move(replacement)), prepend_(prepend), first_only_(first_only), split_(split) {}
+ bool PrependsFirstOnly() const override { return prepend_ && first_only_; }
+ void Split(std::string_view text, std::vector<std::string>& out, bool origin) const override {
if (text.empty()) return;
std::string source = ReplaceAllLiteral(std::string(text), " ", replacement_);
- if (prepend_ && source.compare(0, replacement_.size(), replacement_) != 0) source = replacement_ + source;
+ if (prepend_ && (origin || !first_only_) && source.compare(0, replacement_.size(), replacement_) != 0) source = replacement_ + source;
if (!split_) {
out.push_back(source);
return;
@@ -898,17 +936,24 @@ class Metaspace : public PreTokenizer {
private:
std::string replacement_;
- bool prepend_, split_;
+ bool prepend_, first_only_, split_;
};
class SequencePre : public PreTokenizer {
public:
std::vector<std::unique_ptr<PreTokenizer>> steps;
- void Split(std::string_view text, std::vector<std::string>& out) const override {
+ bool PrependsFirstOnly() const override {
+ for (const auto& step : steps) {
+ if (step->PrependsFirstOnly()) return true;
+ }
+ return false;
+ }
+ void Split(std::string_view text, std::vector<std::string>& out, bool origin) const override {
std::vector<std::string> current{std::string(text)};
for (const auto& step : steps) {
std::vector<std::string> next;
- for (const auto& piece : current) step->Split(piece, next);
+ // Every pre-tokenizer here keeps the text whole, so only the first piece is still at the start.
+ for (size_t i = 0; i < current.size(); i++) step->Split(current[i], next, origin && i == 0);
current.swap(next);
}
out.insert(out.end(), current.begin(), current.end());
@@ -924,8 +969,13 @@ std::unique_ptr<PreTokenizer> ParsePreTokenizer(const Value* n) {
if (ud::RuneCount(replacement) != 1) Refuse("a Metaspace replacement must be exactly one code point");
std::string scheme = Str(*n, "prepend_scheme", "always");
bool split = Bool(*n, "split", true);
- if (scheme == "always" || scheme == "first") return std::make_unique<Metaspace>(replacement, true, split);
- if (scheme == "never") return std::make_unique<Metaspace>(replacement, false, split);
+ // The legacy flag must agree with the scheme, as in the crate (true with "never" is accepted).
+ if (!Bool(*n, "add_prefix_space", true) && scheme != "never") {
+ Refuse("a Metaspace add_prefix_space does not match its declared prepend_scheme");
+ }
+ if (scheme == "always") return std::make_unique<Metaspace>(replacement, true, false, split);
+ if (scheme == "first") return std::make_unique<Metaspace>(replacement, true, true, split);
+ if (scheme == "never") return std::make_unique<Metaspace>(replacement, false, false, split);
Refuse("unknown Metaspace prepend_scheme \"" + scheme + "\"");
}
if (kind == "Sequence") {
@@ -1124,6 +1174,7 @@ struct Span {
std::string text;
int32_t id = 0;
bool is_token = false;
+ size_t offset = 0; // byte offset of the span in the text that was cut
};
// The added tokens that start with one character, longest first (the reference's leftmost-longest
@@ -1166,13 +1217,18 @@ class AddedVocab {
std::vector<Span> out;
if (text.empty()) return out;
if (empty) {
- out.push_back(Span{text, 0, false});
+ out.push_back(Span{text, 0, false, 0});
return out;
}
size_t plain_from = 0;
for (size_t at = 0; at < text.size();) {
auto d = ud::DecodeRune(std::string_view(text).substr(at));
- const AddedToken* hit = LongestAt(text, at, d.rune, d.width);
+ size_t skip = 0;
+ const AddedToken* hit = LongestAt(text, at, d.rune, d.width, &skip);
+ if (skip != 0) { // the crate skips a longest hit that fails single_word, shorter tokens inside it too
+ at += skip;
+ continue;
+ }
if (hit == nullptr) {
at += d.width;
continue;
@@ -1192,11 +1248,11 @@ class AddedVocab {
stop += nx.width;
}
}
- if (start > plain_from) out.push_back(Span{text.substr(plain_from, start - plain_from), 0, false});
- out.push_back(Span{hit->content, hit->id, true});
+ if (start > plain_from) out.push_back(Span{text.substr(plain_from, start - plain_from), 0, false, plain_from});
+ out.push_back(Span{hit->content, hit->id, true, start});
plain_from = at = stop;
}
- if (plain_from < text.size()) out.push_back(Span{text.substr(plain_from), 0, false});
+ if (plain_from < text.size()) out.push_back(Span{text.substr(plain_from), 0, false, plain_from});
return out;
}
@@ -1206,7 +1262,8 @@ class AddedVocab {
auto it = other_.find(r);
return it == other_.end() ? nullptr : &it->second;
}
- const AddedToken* LongestAt(const std::string& text, size_t at, char32_t first, size_t width) const {
+ // `skip` is set to the length of a longest hit that fails single_word.
+ const AddedToken* LongestAt(const std::string& text, size_t at, char32_t first, size_t width, size_t* skip) const {
const Bucket* b = BucketOf(first);
if (b == nullptr || b->tokens.empty()) return nullptr;
if (!b->whole) {
@@ -1215,7 +1272,10 @@ class AddedVocab {
}
for (const AddedToken* t : b->tokens) {
if (text.compare(at, t->content.size(), t->content) != 0) continue;
- if (t->single_word && !StandsAlone(text, at, at + t->content.size())) continue;
+ if (t->single_word && !StandsAlone(text, at, at + t->content.size())) {
+ *skip = t->content.size();
+ return nullptr;
+ }
return t;
}
return nullptr;
@@ -1280,6 +1340,13 @@ class Bpe {
Refuse("merge at rank " + std::to_string(rank) + " produces \"" + pair.first + pair.second +
"\", which the vocabulary does not contain");
}
+ // The crate builds byte-fallback and unknown tokens before merging; this port applies them after.
+ for (const std::string* name : {&pair.first, &pair.second, &merged->first}) {
+ unsigned char b;
+ if ((byte_fallback && ByteToken(*name, &b)) || (unk != nullptr && *name == *unk)) {
+ Unsupported("merge at rank " + std::to_string(rank) + " names the byte or unknown token \"" + *name + "\"");
+ }
+ }
merges_.emplace(PairKey(l->second, r->second), MergeRule{int32_t(rank), merged->second}); // the lowest rank wins
}
if (byte_fallback) {
@@ -1517,8 +1584,13 @@ std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value*
"\". Refusing to load rather than mis-tokenize silently");
}
auto impl = std::make_shared<Impl>();
- impl->normalizer = ParseNormalizer(Member(doc, "normalizer"));
+ bool normalizer_may_drop = false;
+ impl->normalizer = ParseNormalizer(Member(doc, "normalizer"), &normalizer_may_drop);
impl->pre = ParsePreTokenizer(Member(doc, "pre_tokenizer"));
+ if (normalizer_may_drop && impl->pre->PrependsFirstOnly()) {
+ // The crate decides "first" by the original offset of the normalized text, which this port does not track.
+ Unsupported("a Metaspace prepend_scheme \"first\" after a normalizer that can delete leading text (Strip, Replace)");
+ }
impl->decoder = ParseDecoder(Member(doc, "decoder"));
impl->byte_level_only = dynamic_cast<const ByteLevel*>(impl->pre.get()) != nullptr;
@@ -1541,6 +1613,12 @@ std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value*
Refuse("added token id " + std::to_string(i) + " is claimed by both \"" + clash->second + "\" and \"" + content + "\"");
}
seen[i] = content;
+ for (const char* flag : {"special", "lstrip", "rstrip", "single_word", "normalized"}) {
+ const Value* f = r.Find(flag);
+ if (f == nullptr || f->kind() != Value::Kind::Bool) {
+ Refuse("added token \"" + content + "\" must declare " + flag + " as a boolean");
+ }
+ }
added.push_back(AddedToken{content, i, Bool(r, "special", false), Bool(r, "lstrip", false), Bool(r, "rstrip", false),
Bool(r, "single_word", false), Bool(r, "normalized", false)});
}
@@ -1555,6 +1633,18 @@ std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value*
if (v.as_int() < 0) Refuse("the vocabulary holds a negative id " + std::to_string(v.as_int()));
vocab[vocab_v->key(i)] = int32_t(v.as_int());
}
+ {
+ int32_t next = int32_t(vocab.size());
+ for (auto& tok : added) {
+ auto known = vocab.find(tok.content);
+ int32_t expect = known != vocab.end() ? known->second : next++;
+ if (known == vocab.end()) vocab[tok.content] = expect;
+ if (tok.id != expect) {
+ Refuse("added token \"" + tok.content + "\" declares id " + std::to_string(tok.id) + ", but the reference gives it " +
+ std::to_string(expect) + " (its vocabulary id, or the next free id in file order)");
+ }
+ }
+ }
for (const auto& tok : added) {
if (tok.special) {
impl->is_special.insert(tok.id);
@@ -1580,6 +1670,13 @@ std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value*
std::vector<std::pair<std::string, std::string>> merges;
ReadMerges(model, merges);
+ for (const char* key : {"end_of_word_suffix", "continuing_subword_prefix"}) {
+ const Value* v = Member(model, key);
+ if (v != nullptr && v->kind() == Value::Kind::String && !v->as_string().empty()) {
+ Unsupported(std::string("model.") + key + " is not implemented");
+ }
+ }
+ if (Member(model, "dropout") != nullptr) Unsupported("model.dropout is not implemented");
std::string unk;
const Value* unk_v = Member(model, "unk_token");
if (unk_v != nullptr && unk_v->kind() != Value::Kind::String) Refuse("model.unk_token must be a string or null");
@@ -1689,9 +1786,9 @@ std::vector<int32_t> Tokenizer::Encode(std::string_view text_in, int64_t limit)
std::string text = ud::ToValid(text_in); // a Python str is always valid
out.reserve(size_t(limit > 0 && limit < int64_t(text.size() / 3 + 16) ? limit : int64_t(text.size() / 3 + 16)));
auto full = [&]() { return limit >= 0 && int64_t(out.size()) >= limit; };
- auto plain = [&](const std::string& s) {
+ auto plain = [&](const std::string& s, bool origin) {
std::vector<std::string> pieces;
- t.pre->Split(s, pieces);
+ t.pre->Split(s, pieces, origin);
for (const auto& piece : pieces) {
if (full()) return;
t.bpe->EncodePiece(piece, out);
@@ -1704,14 +1801,15 @@ std::vector<int32_t> Tokenizer::Encode(std::string_view text_in, int64_t limit)
continue;
}
std::string normalized = t.normalizer(sp.text);
+ bool origin = sp.offset == 0; // only the text at the start of the input is "first"
if (t.normal_added.empty) {
- plain(normalized);
+ plain(normalized, origin);
continue;
}
for (const Span& inner : t.normal_added.Split(normalized)) {
if (full()) break;
if (inner.is_token) out.push_back(inner.id);
- else plain(inner.text);
+ else plain(inner.text, origin && inner.offset == 0);
}
}
if (limit >= 0 && int64_t(out.size()) > limit) out.resize(size_t(limit));
@@ -1768,7 +1866,7 @@ std::string Tokenizer::MaskToken() const { return impl_->mask_token; }
size_t Tokenizer::CountMergeDisagreements(std::string_view text, size_t* pieces_checked) const {
size_t bad = 0;
std::vector<std::string> pieces;
- impl_->pre->Split(impl_->normalizer(ud::ToValid(text)), pieces);
+ impl_->pre->Split(impl_->normalizer(ud::ToValid(text)), pieces, true);
for (const auto& p : pieces) {
if (!impl_->bpe->StrategiesAgree(p)) bad++;
if (pieces_checked != nullptr) (*pieces_checked)++;
@@ -1779,7 +1877,7 @@ size_t Tokenizer::CountMergeDisagreements(std::string_view text, size_t* pieces_
std::string Tokenizer::NormalizeForTest(std::string_view text) const { return impl_->normalizer(ud::ToValid(text)); }
std::vector<std::string> Tokenizer::PreTokenizeForTest(std::string_view normalized) const {
std::vector<std::string> out;
- impl_->pre->Split(normalized, out);
+ impl_->pre->Split(normalized, out, true);
return out;
}
@@ -1801,7 +1899,7 @@ Result<std::vector<std::string>> RunPreTokenizerSpec(std::string_view spec_json,
auto v = pyvalue::Parse(spec_json);
if (!v.ok()) return Result<std::vector<std::string>>::Fail(v.error());
std::vector<std::string> out;
- ParsePreTokenizer(&v.value())->Split(input, out);
+ ParsePreTokenizer(&v.value())->Split(input, out, true);
return Result<std::vector<std::string>>::Ok(out);
} catch (const Fail& f) {
return Result<std::vector<std::string>>::Fail(f.msg);
diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cpp
index b496077a..ca59a222 100644
--- a/laya-cpp/tests/tokenizer_test.cpp
+++ b/laya-cpp/tests/tokenizer_test.cpp
@@ -1,6 +1,7 @@
// laya::tokenizer against the `tokenizers` crate (through the Python-recorded goldens) and against
// hand-built tokenizers. Ported from laya-go/tokenizer's tests.
#include <algorithm>
+#include <chrono>
#include <cstdlib>
#include <fstream>
#include <mutex>
@@ -86,6 +87,17 @@ Tokenizer MustParse(const std::string& json, const char* config = nullptr) {
return std::move(t).value();
}
+// An added_tokens entry with every field the crate requires; `on` names the flags that are true.
+std::string Tok(const std::string& content, int id, std::initializer_list<const char*> on = {}) {
+ std::string out = "{\"content\":\"" + content + "\",\"id\":" + std::to_string(id);
+ for (const char* flag : {"special", "lstrip", "rstrip", "single_word", "normalized"}) {
+ bool v = false;
+ for (const char* o : on) v = v || std::string(o) == flag;
+ out += std::string(",\"") + flag + "\":" + (v ? "true" : "false");
+ }
+ return out + "}";
+}
+
// A tiny byte-level tokenizer built in the test, so the loader is checked without a checkpoint.
std::string ByteLevelSpec(const std::string& extra) {
std::string vocab;
@@ -181,11 +193,11 @@ TEST(TokenizerNormalize, LowercaseMatchesTheCrateOverAllUnicode) {
TEST(TokenizerAdded, SingleWordFlankMatchesTheCrate) {
LAYA_GOLDEN_LIST(sw, "go_extra.json", "single_word");
Tokenizer tok = MustParse(
- R"({"model":{"type":"BPE","vocab":{"a":0,"<unk>":2},"merges":[],"unk_token":"<unk>"},
- "added_tokens":[{"content":"TOK","id":1,"single_word":true,"special":true}]})");
+ R"({"model":{"type":"BPE","vocab":{"a":0,"<unk>":1},"merges":[],"unk_token":"<unk>"},
+ "added_tokens":[{"content":"TOK","id":2,"single_word":true,"special":true,"lstrip":false,"rstrip":false,"normalized":false}]})");
auto standalone = [&](const std::string& s) {
for (int32_t id : tok.Encode(s)) {
- if (id == 1) return std::string("1");
+ if (id == 2) return std::string("1");
}
return std::string("0");
};
@@ -478,7 +490,7 @@ TEST(TokenizerLoader, Refuses) {
{"a merge that is not a pair", Replace(good, "\"a b\"", "[\"a\"]"), false},
{"a space-joined merge with an empty half", Replace(good, "\"a b\"", "\"a \""), false},
{"an added token with no id", ByteLevelSpec(",\"added_tokens\":[{\"content\":\"x\"}]"), false},
- {"two added tokens sharing an id", ByteLevelSpec(",\"added_tokens\":[{\"content\":\"x\",\"id\":300},{\"content\":\"y\",\"id\":300}]"), false},
+ {"two added tokens sharing an id", ByteLevelSpec(",\"added_tokens\":[" + Tok("x", 257) + "," + Tok("y", 257) + "]"), false},
{"a vocabulary giving one id to two tokens", Replace(good, "\"ab\":256", "\"ab\":0"), false},
{"no vocabulary", "{\"model\":{\"type\":\"BPE\",\"merges\":[]}}", false},
{"not JSON", "{", false},
@@ -520,26 +532,24 @@ TEST(TokenizerLoader, IdsFarBeyondTheVocabularyAreRefused) {
TEST(TokenizerAdded, LongestLstripRstripSingleWordNormalized) {
Tokenizer tok = MustParse(ByteLevelSpec(
- ",\"added_tokens\":[{\"content\":\"<S>\",\"id\":300,\"special\":true,\"normalized\":false},"
- "{\"content\":\"<SS>\",\"id\":301,\"special\":true,\"normalized\":false},"
- "{\"content\":\"[M]\",\"id\":302,\"special\":true,\"lstrip\":true,\"rstrip\":true},"
- "{\"content\":\"word\",\"id\":303,\"single_word\":true},"
- "{\"content\":\"\xC3\xA9" "clair\",\"id\":304,\"normalized\":true},"
- "{\"content\":\"\xC2\xA7\",\"id\":305},{\"content\":\"!\",\"id\":306}],\"decoder\":{\"type\":\"ByteLevel\"}"));
+ ",\"added_tokens\":[" + Tok("<S>", 257, {"special"}) + "," + Tok("<SS>", 258, {"special"}) + "," +
+ Tok("[M]", 259, {"special", "lstrip", "rstrip"}) + "," + Tok("word", 260, {"single_word"}) + "," +
+ Tok("\xC3\xA9" "clair", 261, {"normalized"}) + "," + Tok("\xC2\xA7", 167) + "," + Tok("!", 33) +
+ "],\"decoder\":{\"type\":\"ByteLevel\"}"));
using V = std::vector<int32_t>;
- EXPECT_EQ(tok.Encode("<SS><S>"), (V{301, 300})) << "the longest token wins at a position";
- EXPECT_EQ(tok.Encode("a [M] b"), (V{97, 302, 98})) << "lstrip and rstrip absorb the white space";
- EXPECT_EQ(tok.Encode("word"), (V{303}));
+ EXPECT_EQ(tok.Encode("<SS><S>"), (V{258, 257})) << "the longest token wins at a position";
+ EXPECT_EQ(tok.Encode("a [M] b"), (V{97, 259, 98})) << "lstrip and rstrip absorb the white space";
+ EXPECT_EQ(tok.Encode("word"), (V{260}));
auto inside = tok.Encode("swords word.");
ASSERT_GE(inside.size(), 3u);
- EXPECT_EQ(inside[inside.size() - 2], 303) << "a single_word token inside a word is not matched but a free one is";
- EXPECT_EQ(std::count(inside.begin(), inside.end(), 303), 1) << "swords holds the word but not as a word";
- EXPECT_EQ(tok.Encode("\xC3\xA9" "clair"), (V{304}));
- EXPECT_EQ(tok.Encode("a\xC2\xA7" "b!"), (V{97, 305, 98, 306}));
+ EXPECT_EQ(inside[inside.size() - 2], 260) << "a single_word token inside a word is not matched but a free one is";
+ EXPECT_EQ(std::count(inside.begin(), inside.end(), 260), 1) << "swords holds the word but not as a word";
+ EXPECT_EQ(tok.Encode("\xC3\xA9" "clair"), (V{261}));
+ EXPECT_EQ(tok.Encode("a\xC2\xA7" "b!"), (V{97, 167, 98, 33})) << "added tokens already in the vocabulary keep its ids";
EXPECT_EQ(tok.Encode("<S"), (V{60, 83})) << "a prefix of a token is not the token";
- EXPECT_EQ(tok.Decode({300, 97, 301}, true), "a");
- EXPECT_EQ(tok.Decode({300, 97}, false), "<S>a");
- EXPECT_EQ(tok.SpecialId("<S>"), std::optional<int32_t>(300));
+ EXPECT_EQ(tok.Decode({257, 97, 258}, true), "a");
+ EXPECT_EQ(tok.Decode({257, 97}, false), "<S>a");
+ EXPECT_EQ(tok.SpecialId("<S>"), std::optional<int32_t>(257));
EXPECT_FALSE(tok.SpecialId("word").has_value());
EXPECT_EQ(tok.SpecialTokens().size(), 3u);
}
@@ -551,13 +561,13 @@ TEST(TokenizerLoader, RolesFromTokenizerConfigAndDirectoryLayout) {
};
std::string cmd = "mkdir -p '" + base + "/m/tokenizer' '" + base + "/plain' '" + base + "/empty'";
ASSERT_EQ(std::system(cmd.c_str()), 0);
- std::string spec = ByteLevelSpec(",\"added_tokens\":[{\"content\":\"<c>\",\"id\":300,\"special\":true},{\"content\":\"<m>\",\"id\":301,\"special\":true}]");
+ std::string spec = ByteLevelSpec(",\"added_tokens\":[" + Tok("<c>", 257, {"special"}) + "," + Tok("<m>", 258, {"special"}) + "]");
write(base + "/plain/tokenizer.json", spec);
write(base + "/plain/tokenizer_config.json", "{\"cls_token\":\"<c>\",\"mask_token\":{\"content\":\"<m>\"},\"pad_token\":null}");
auto t = Tokenizer::LoadDir(base + "/plain");
ASSERT_TRUE(t.ok()) << t.error();
- EXPECT_EQ(t.value().Cls(), std::optional<int32_t>(300));
- EXPECT_EQ(t.value().Mask(), std::optional<int32_t>(301));
+ EXPECT_EQ(t.value().Cls(), std::optional<int32_t>(257));
+ EXPECT_EQ(t.value().Mask(), std::optional<int32_t>(258));
EXPECT_EQ(t.value().MaskToken(), "<m>");
EXPECT_FALSE(t.value().Pad().has_value()) << "a null role is absent";
EXPECT_FALSE(t.value().Sep().has_value()) << "an unnamed role is absent";
@@ -696,8 +706,8 @@ TEST(TokenizerRefusals, SequenceRunsInOrder) {
}
TEST(TokenizerAdded, RstripAbsorbsAllWhiteSpace) {
- Tokenizer tok = MustParse(ByteLevelSpec(",\"added_tokens\":[{\"content\":\"[M]\",\"id\":302,\"special\":true,\"rstrip\":true}]"));
- EXPECT_EQ(tok.Encode("[M] \t b"), (std::vector<int32_t>{302, 98}));
+ Tokenizer tok = MustParse(ByteLevelSpec(",\"added_tokens\":[" + Tok("[M]", 257, {"special", "rstrip"}) + "]"));
+ EXPECT_EQ(tok.Encode("[M] \t b"), (std::vector<int32_t>{257, 98}));
}
// Canonical ordering must run before composition: a + dot above (230) + dot below (220) is a-dot-below
@@ -707,3 +717,262 @@ TEST(TokenizerNormalize, MarksAreReorderedBeforeTheyCompose) {
EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFKC), "\xE1\xBA\xA1\xCC\x87");
EXPECT_EQ(tk::Normalize("a\xCC\x87\xCC\xA3", Form::NFD), "a\xCC\xA3\xCC\x87");
}
+
+// ---- review round 1: every case's expected value is the crate's (tokenizers 0.23.2) ---------------------
+namespace {
+
+std::string NormSpec(const std::string& pattern, const std::string& to = "_") {
+ return "{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"" + pattern + "\"},\"content\":\"" + to + "\"}";
+}
+
+// Letters a-z are 0-25, the marker is 26, <unk> is 27 and the first added token is 28.
+std::string MetaspaceSpec(const std::string& pre, const std::string& normalizer = "null") {
+ std::string vocab;
+ for (char c = 'a'; c <= 'z'; c++) vocab += std::string("\"") + c + "\":" + std::to_string(c - 'a') + ",";
+ vocab += "\"\xE2\x96\x81\":26,\"<unk>\":27";
+ return "{\"model\":{\"type\":\"BPE\",\"vocab\":{" + vocab + "},\"merges\":[],\"unk_token\":\"<unk>\"},\"normalizer\":" + normalizer +
+ ",\"pre_tokenizer\":" + pre + ",\"added_tokens\":[" + Tok("<m>", 28, {"special"}) + "]}";
+}
+
+std::string Metaspace(const char* scheme, bool split = true) {
+ return std::string("{\"type\":\"Metaspace\",\"replacement\":\"\xE2\x96\x81\",\"prepend_scheme\":\"") + scheme + "\",\"split\":" +
+ (split ? "true" : "false") + "}";
+}
+
+using Ids32 = std::vector<int32_t>;
+
+} // namespace
+
+TEST(TokenizerRegex, RepeatsAreBoundedByProgramSizeAndDepth) {
+ // (((a{1000}){1000}){10})x built about 450 MB of program before the cap.
+ for (const char* p : {"(((a{1000}){1000}){30})x", "((a{300}){300}){2}"}) {
+ auto t0 = std::chrono::steady_clock::now();
+ auto r = tk::RunNormalizerSpec(NormSpec(p), "aaa");
+ auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0).count();
+ ASSERT_FALSE(r.ok()) << p;
+ EXPECT_FALSE(tk::IsUnsupported(r.error())) << r.error();
+ EXPECT_LT(ms, 600) << p << " was expanded before it was refused";
+ }
+ EXPECT_TRUE(tk::RunNormalizerSpec(NormSpec("a{1000}b{1000}"), "ab").ok()) << "a large but ordinary repeat still loads";
+ auto deep = [](size_t n) { return std::string(n, '(') + "a" + std::string(n, ')'); };
+ EXPECT_EQ(tk::RunNormalizerSpec(NormSpec(deep(100)), "baab").value(), "b__b") << "the crate gives b__b";
+ EXPECT_TRUE(tk::RunNormalizerSpec(NormSpec(deep(256)), "a").ok());
+ EXPECT_FALSE(tk::RunNormalizerSpec(NormSpec(deep(257)), "a").ok());
+ EXPECT_FALSE(tk::RunNormalizerSpec(NormSpec(deep(200000)), "a").ok()) << "was a stack overflow";
+ std::string siblings;
+ for (int i = 0; i < 300; i++) siblings += "(a)";
+ EXPECT_EQ(tk::RunNormalizerSpec(NormSpec(siblings), "a").value(), "a") << "groups side by side are not nested";
+ std::string wide;
+ for (int i = 0; i < 60000; i++) wide += "a|";
+ EXPECT_FALSE(tk::RunNormalizerSpec(NormSpec(wide + "b"), "a").ok()) << "120000 instructions";
+}
+
+TEST(TokenizerRegex, ReplaceAllIsLinearInTheText) {
+ std::string text(400000, 'a');
+ auto t0 = std::chrono::steady_clock::now();
+ auto r = tk::RunNormalizerSpec(NormSpec("a"), text);
+ auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0).count();
+ ASSERT_TRUE(r.ok());
+ EXPECT_EQ(r.value(), std::string(400000, '_'));
+ EXPECT_LT(ms, 3000) << "a table sized by the whole text was allocated per match";
+}
+
+TEST(TokenizerRegex, SparseTableAgreesWithTheCrate) {
+ // A program big enough that (instruction x position) exceeds the dense table uses the sparse one.
+ std::string big(100000, 'b');
+ auto s = tk::RunNormalizerSpec(NormSpec("(x{1000}y{1000}z{1000}|b)"), big + "x");
+ ASSERT_TRUE(s.ok()) << s.error();
+ EXPECT_EQ(s.value(), std::string(100000, '_') + "x");
+ // Without the memo this is exponential, and it must also hold on the sparse table.
+ auto fib = tk::RunNormalizerSpec(NormSpec("(a|aa)+b|x{1000}y{1000}z{1000}"), std::string(5000, 'a') + std::string(95000, 'c') + "x");
+ ASSERT_TRUE(fib.ok()) << fib.error();
+ EXPECT_EQ(fib.value(), std::string(5000, 'a') + std::string(95000, 'c') + "x");
+ // Matches after the first share the table; each must still be found.
+ EXPECT_EQ(tk::RunNormalizerSpec(NormSpec("a+?b?|b"), "aab ab b").value(), "__ _ _");
+}
+
+TEST(TokenizerRegex, CaseInsensitiveMatchesTheCrateOrIsRefused) {
+ auto run = [](const std::string& pattern, const std::string& in) {
+ auto r = tk::RunNormalizerSpec(NormSpec(pattern), in);
+ EXPECT_TRUE(r.ok()) << pattern << ": " << r.error();
+ return r.ok() ? r.value() : std::string("<error>");
+ };
+ EXPECT_EQ(run("(?i)[a-c]+", "AbCd"), "_d");
+ EXPECT_EQ(run("(?i)[^a-c]", "aBcD"), "aBc_");
+ EXPECT_EQ(run("(?i)[A-C]", "abcABC"), "______");
+ EXPECT_EQ(run("(?i)x[a-c]y", "XAY xcY xdy"), "_ _ xdy");
+ EXPECT_EQ(run("(?i)q|Z", "qQzZ"), "____");
+ // The crate folds U+212A (Kelvin) to k, U+017F and U+00DF to s and ss, and ligatures to letter pairs.
+ for (const char* p : {"(?i)k", "(?i)K", "(?i)ss", "(?i)[a-z]", "(?i)[^k]", "(?i)f", "(?i)fi", "(?i)(?:s)t", "(?i)[k-k]"}) {
+ auto r = tk::RunNormalizerSpec(NormSpec(p), "a");
+ ASSERT_FALSE(r.ok()) << p;
+ EXPECT_TRUE(tk::IsUnsupported(r.error())) << p << ": " << r.error();
+ }
+ EXPECT_EQ(run("k", "kK"), "_K") << "case-sensitive k is not affected";
+ EXPECT_EQ(run("ss", "ssSs"), "_Ss");
+}
+
+TEST(TokenizerPre, MetaspaceFirstPrependsOnlyAtTheStartOfTheInput) {
+ struct Row {
+ const char* scheme;
+ const char* text;
+ Ids32 want;
+ } rows[] = {
+ {"first", "a<m>b c", {26, 0, 28, 1, 26, 2}}, {"first", "<m>b c", {28, 1, 26, 2}},
+ {"first", "b", {26, 1}}, {"first", " <m>b", {26, 28, 1}},
+ {"first", "<m><m>b", {28, 28, 1}}, {"first", "<m>b<m>c", {28, 1, 28, 2}},
+ {"first", "a<m>b<m>c", {26, 0, 28, 1, 28, 2}}, {"first", "a b<m> c", {26, 0, 26, 1, 28, 26, 2}},
+ {"always", "a<m>b c", {26, 0, 28, 26, 1, 26, 2}}, {"always", "<m>b c", {28, 26, 1, 26, 2}},
+ {"never", "a<m>b c", {0, 28, 1, 26, 2}}, {"never", "b", {1}},
+ };
+ for (const auto& row : rows) {
+ Tokenizer tok = MustParse(MetaspaceSpec(Metaspace(row.scheme)));
+ EXPECT_EQ(tok.Encode(row.text), row.want) << row.scheme << " " << row.text;
+ }
+ // Only the first piece of an earlier split is still at the start.
+ Tokenizer bytes = MustParse(MetaspaceSpec("{\"type\":\"Sequence\",\"pretokenizers\":[{\"type\":\"ByteLevel\",\"add_prefix_space\":false}," +
+ Metaspace("first") + "]}"));
+ EXPECT_EQ(bytes.Encode("ab cd"), (Ids32{26, 0, 1, 27, 2, 3}));
+ EXPECT_EQ(bytes.Encode("ab12 cd"), (Ids32{26, 0, 1, 27, 27, 27, 2, 3}));
+ Tokenizer seq = MustParse(MetaspaceSpec("{\"type\":\"Sequence\",\"pretokenizers\":[" + Metaspace("first") + "," + Metaspace("first") + "]}"));
+ EXPECT_EQ(seq.Encode("ab<m>ab"), (Ids32{26, 0, 1, 28, 0, 1}));
+ Tokenizer nosplit = MustParse(MetaspaceSpec(Metaspace("first", false)));
+ EXPECT_EQ(nosplit.Encode("a b<m>c d"), (Ids32{26, 0, 26, 1, 28, 2, 26, 3}));
+ // The normalizer runs on each piece; the second piece is still not at the start.
+ Tokenizer prepend = MustParse(MetaspaceSpec(Metaspace("first"), R"({"type":"Prepend","prepend":"x"})"));
+ EXPECT_EQ(prepend.Encode("a<m>b"), (Ids32{26, 23, 0, 28, 23, 1}));
+ // The crate keys "first" on the original offset, which a normalizer that deletes text changes.
+ auto strip = Tokenizer::Parse(MetaspaceSpec(Metaspace("first"), R"({"type":"Strip","strip_left":true,"strip_right":false})"));
+ ASSERT_FALSE(strip.ok());
+ EXPECT_TRUE(tk::IsUnsupported(strip.error())) << strip.error();
+ EXPECT_TRUE(Tokenizer::Parse(MetaspaceSpec(Metaspace("always"), R"({"type":"Strip","strip_left":true,"strip_right":false})")).ok());
+ auto replace = Tokenizer::Parse(MetaspaceSpec(Metaspace("first"), R"({"type":"Replace","pattern":{"String":"z"},"content":""})"));
+ ASSERT_FALSE(replace.ok());
+ EXPECT_TRUE(tk::IsUnsupported(replace.error())) << replace.error();
+ EXPECT_EQ(MustParse(MetaspaceSpec(Metaspace("first"), R"({"type":"Strip","strip_left":false,"strip_right":true})")).Encode("ab "),
+ (Ids32{26, 0, 1})) << "a right strip leaves the start alone";
+ // An added token matched in the normalized text: its pieces are still not at the start.
+ std::string normalized_token = MetaspaceSpec(Metaspace("first"));
+ normalized_token.replace(normalized_token.find("\"normalized\":false"), 18, "\"normalized\":true");
+ Tokenizer inner = MustParse(normalized_token);
+ EXPECT_EQ(inner.Encode("a<m>b c"), (Ids32{26, 0, 28, 1, 26, 2}));
+ EXPECT_EQ(inner.Encode("<m>b c"), (Ids32{28, 1, 26, 2}));
+ EXPECT_EQ(inner.Encode(" <m>b"), (Ids32{26, 28, 1}));
+}
+
+TEST(TokenizerPre, MetaspaceLegacyFlagMustAgreeWithTheScheme) {
+ auto spec = [](const std::string& tail) {
+ return MetaspaceSpec("{\"type\":\"Metaspace\",\"replacement\":\"\xE2\x96\x81\"" + tail + "}");
+ };
+ // The crate: false with any scheme but "never" (or none) is an error; true with "never" is accepted.
+ for (const char* tail : {",\"add_prefix_space\":false", ",\"add_prefix_space\":false,\"prepend_scheme\":\"always\"",
+ ",\"add_prefix_space\":false,\"prepend_scheme\":\"first\""}) {
+ EXPECT_FALSE(Tokenizer::Parse(spec(tail)).ok()) << tail;
+ }
+ EXPECT_EQ(MustParse(spec(",\"add_prefix_space\":false,\"prepend_scheme\":\"never\"")).Encode("ab"), (Ids32{0, 1}));
+ EXPECT_EQ(MustParse(spec(",\"add_prefix_space\":true,\"prepend_scheme\":\"never\"")).Encode("ab"), (Ids32{0, 1}));
+ EXPECT_EQ(MustParse(spec(",\"add_prefix_space\":true")).Encode("ab"), (Ids32{26, 0, 1}));
+ EXPECT_EQ(MustParse(spec("")).Encode("ab"), (Ids32{26, 0, 1}));
+}
+
+TEST(TokenizerAdded, SingleWordFailureSkipsTheWholeLongestHit) {
+ // xyz is single_word, xy is free. The crate drops a longest hit that fails the flank test and resumes
+ // after it, so xy inside "wxyz" is not found.
+ Tokenizer tok = MustParse(ByteLevelSpec(",\"added_tokens\":[" + Tok("xyz", 257, {"single_word"}) + "," + Tok("xy", 258) + "," + Tok("yz", 259) + "]"));
+ struct Row {
+ const char* text;
+ Ids32 want;
+ } rows[] = {
+ {"wxyz", {119, 120, 121, 122}},
+ {"xyz", {257}},
+ {"xyzw xyz", {120, 121, 122, 119, 32, 257}},
+ {"xyzw xy", {120, 121, 122, 119, 32, 258}},
+ {"w xyz w", {119, 32, 257, 32, 119}},
+ {"xyz,wxyz", {257, 44, 119, 120, 121, 122}},
+ {"wxyz yz", {119, 120, 121, 122, 32, 259}}, // yz inside the failed xyz is not found either
+ {"xy", {258}},
+ {"wxy", {119, 258}},
+ };
+ for (const auto& row : rows) EXPECT_EQ(tok.Encode(row.text), row.want) << row.text;
+}
+
+TEST(TokenizerLoader, AddedTokenIdsFollowTheCrate) {
+ // The crate keeps the vocabulary id of an added token already in the vocabulary and numbers the others
+ // from the vocabulary size in file order; the id in the file is not used. Ids that differ are refused.
+ auto load = [](const std::string& tokens) { return Tokenizer::Parse(ByteLevelSpec(",\"added_tokens\":[" + tokens + "]")); };
+ EXPECT_FALSE(load(Tok("ab", 300, {"special"})).ok()) << "ab is vocabulary id 256 in the crate";
+ EXPECT_TRUE(load(Tok("ab", 256, {"special"})).ok());
+ EXPECT_FALSE(load(Tok("<x>", 300)).ok()) << "the crate numbers it 257";
+ EXPECT_FALSE(load(Tok("<x>", 257) + "," + Tok("<y>", 259)).ok()) << "the crate numbers y 258";
+ EXPECT_TRUE(load(Tok("<x>", 257) + "," + Tok("ab", 256) + "," + Tok("<y>", 258)).ok()) << "a vocabulary token takes no new id";
+ auto r = load(Tok("<x>", 300));
+ EXPECT_NE(r.error().find("257"), std::string::npos) << r.error();
+ // The crate needs all five flags as booleans.
+ for (const char* flag : {"special", "lstrip", "rstrip", "single_word", "normalized"}) {
+ std::string entry = Tok("<x>", 257);
+ std::string drop = std::string(",\"") + flag + "\":false";
+ entry.erase(entry.find(drop), drop.size());
+ EXPECT_FALSE(load(entry).ok()) << "missing " << flag;
+ std::string null = Tok("<x>", 257);
+ null.replace(null.find(drop), drop.size(), std::string(",\"") + flag + "\":null");
+ EXPECT_FALSE(load(null).ok()) << "null " << flag;
+ }
+}
+
+TEST(TokenizerLoader, ModelOptionsThePortIgnoresAreRefused) {
+ std::string good = ByteLevelSpec("");
+ auto with = [&](const std::string& field) { return Replace(good, "\"merges\":[\"a b\"]", "\"merges\":[\"a b\"]," + field); };
+ for (const char* f : {"\"end_of_word_suffix\":\"</w>\"", "\"continuing_subword_prefix\":\"##\"", "\"dropout\":0.5", "\"dropout\":0.0"}) {
+ auto r = Tokenizer::Parse(with(f));
+ ASSERT_FALSE(r.ok()) << f;
+ EXPECT_TRUE(tk::IsUnsupported(r.error())) << f;
+ }
+ for (const char* f : {"\"end_of_word_suffix\":\"\"", "\"continuing_subword_prefix\":\"\"", "\"dropout\":null",
+ "\"end_of_word_suffix\":null", "\"continuing_subword_prefix\":null"}) {
+ EXPECT_TRUE(Tokenizer::Parse(with(f)).ok()) << f << " is the same as absent in the crate";
+ }
+}
+
+TEST(TokenizerLoader, MergesNamingByteOrUnknownTokensAreRefused) {
+ // The crate builds byte-fallback and unknown tokens before merging; this port applies them after.
+ auto spec = [](const char* merge, bool byte_fallback, bool unk) {
+ return std::string("{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"b\":1,\"ab\":2,\"<0x61>\":3,\"<unk>\":4,\"<0x61>a\":5,\"a<unk>\":6},") +
+ "\"merges\":[\"a b\"," + merge + "],\"byte_fallback\":" + (byte_fallback ? "true" : "false") +
+ (unk ? ",\"unk_token\":\"<unk>\"" : "") + "}}";
+ };
+ auto byte_merge = spec("\"<0x61> a\"", true, false);
+ auto r = Tokenizer::Parse(byte_merge);
+ ASSERT_FALSE(r.ok());
+ EXPECT_TRUE(tk::IsUnsupported(r.error())) << r.error();
+ auto unk_merge = Tokenizer::Parse(spec("\"a <unk>\"", false, true));
+ ASSERT_FALSE(unk_merge.ok());
+ EXPECT_TRUE(tk::IsUnsupported(unk_merge.error())) << unk_merge.error();
+ EXPECT_TRUE(Tokenizer::Parse(spec("\"<0x61> a\"", false, true)).ok()) << "a byte-shaped name is ordinary without byte_fallback";
+ EXPECT_TRUE(Tokenizer::Parse(spec("\"a <unk>\"", true, false)).ok()) << "<unk> is ordinary when it is not the unknown token";
+ EXPECT_TRUE(Tokenizer::Parse(spec("\"a b\"", true, true)).ok());
+ // A merge that produces a byte token is just as unsound.
+ auto produces = Tokenizer::Parse(
+ "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"<0x6\":1,\"1>\":2,\"<0x61>\":3},\"merges\":[\"<0x6 1>\"],\"byte_fallback\":true}}");
+ ASSERT_FALSE(produces.ok());
+ EXPECT_TRUE(tk::IsUnsupported(produces.error())) << produces.error();
+}
+
+// Three or more branches left stale jump targets (so "a|a|a|b" matched the empty string), and each `|`
+// inserted into the program, which made a long alternation quadratic.
+TEST(TokenizerRegex, ManyBranchAlternation) {
+ auto run = [](const std::string& pattern, const std::string& in) {
+ auto r = tk::RunNormalizerSpec(NormSpec(pattern), in);
+ EXPECT_TRUE(r.ok()) << pattern << ": " << r.error();
+ return r.ok() ? r.value() : std::string("<error>");
+ };
+ EXPECT_EQ(run("a|bc|d|e", "abcdexbd"), "____xb_");
+ EXPECT_EQ(run("(a|b|c)+d", "abcd bd cad"), "_ _ _");
+ EXPECT_EQ(run("x(a|b|c|d)y|z", "xcy xey z"), "_ xey _");
+ EXPECT_EQ(run("a|a|a|b", "cab"), "c__");
+ std::string wide;
+ for (int i = 0; i < 20000; i++) wide += "a|";
+ auto t0 = std::chrono::steady_clock::now();
+ EXPECT_EQ(run(wide + "b", "cab"), "c__");
+ auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0).count();
+ EXPECT_LT(ms, 3000);
+}