Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(cpp): laya::tokenizer, loader, normalizers, pretokenizers, BPE and decoders

Adeshina Lasisicacf753f33e0
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txt
index af2023d2..0fd7830d 100644
--- a/laya-cpp/CMakeLists.txt
+++ b/laya-cpp/CMakeLists.txt
@@ -25,7 +25,8 @@ function(laya_target_flags target)
target_compile_options(${target} PRIVATE ${LAYA_FP_FLAGS} ${LAYA_WARN_FLAGS})
endfunction()
-add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp src/lang.cpp)
+add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp src/lang.cpp
+ src/tokenizer.cpp)
target_include_directories(laya PUBLIC include)
laya_target_flags(laya)
@@ -44,7 +45,8 @@ if(LAYA_BUILD_TESTS)
FetchContent_MakeAvailable(googletest)
enable_testing()
- add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp tests/hardening_test.cpp)
+ add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp tests/hardening_test.cpp
+ tests/tokenizer_test.cpp)
target_link_libraries(laya_tests PRIVATE laya GTest::gtest_main)
laya_target_flags(laya_tests)
diff --git a/laya-cpp/include/laya/tokenizer.hpp b/laya-cpp/include/laya/tokenizer.hpp
new file mode 100644
index 00000000..72325e7c
--- /dev/null
+++ b/laya-cpp/include/laya/tokenizer.hpp
@@ -0,0 +1,81 @@
+#pragma once
+// A reader and runner for Hugging Face tokenizer.json files. It reproduces the ids the reference
+// `tokenizers` crate (0.23.2) gives for the BPE tokenizers laya ships: the normalizer, the
+// pre-tokenizer, the added-token splitter, BPE with its byte fallback, and the decoder chain.
+//
+// Anything the loader does not implement is refused with an error that begins with kUnsupported
+// rather than approximated: an approximated tokenizer changes ids in a way no answer-level test
+// localises. A loaded Tokenizer is immutable and safe for concurrent use. Strings are UTF-8; text
+// that is not valid UTF-8 is read as bytes.decode("utf-8", "replace") would deliver it.
+//
+// The Unicode behaviour that decides ids (letter and number classes, word characters beside a
+// single_word token, lowercase, normalization) is compiled in from the reference, see
+// laya-go/scripts/gen_tables.py --lang cpp, so ids do not depend on the C++ library's Unicode.
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "laya/result.hpp"
+
+namespace laya {
+namespace tokenizer {
+
+// Every error that refuses a feature this library does not implement starts with this text.
+extern const char* const kUnsupported;
+bool IsUnsupported(const std::string& error);
+
+struct Impl;
+
+class Tokenizer {
+ public:
+ Tokenizer() = default; // empty, only so Result<Tokenizer> can hold one; obtain a usable one from Parse or LoadDir
+ // tokenizer.json bytes and, optionally, tokenizer_config.json bytes (names the CLS, SEP, MASK
+ // and PAD tokens).
+ static Result<Tokenizer> Parse(std::string_view tokenizer_json,
+ std::optional<std::string_view> config_json = std::nullopt);
+ // <dir>/tokenizer/tokenizer.json when that exists, else <dir>/tokenizer.json, with the
+ // tokenizer_config.json beside whichever is used (it may be absent).
+ static Result<Tokenizer> LoadDir(const std::string& dir);
+
+ // Token ids of text. limit < 0 means no limit, otherwise only the first limit ids are made.
+ std::vector<int32_t> Encode(std::string_view text, int64_t limit = -1) const;
+ // Ids the vocabulary does not hold are skipped, as the reference skips them; with
+ // skip_special_tokens the added tokens marked special are skipped too.
+ std::string Decode(const std::vector<int32_t>& ids, bool skip_special_tokens) const;
+
+ size_t VocabSize() const; // the model vocabulary plus added tokens outside it
+ int32_t HighestId() const;
+ std::optional<int32_t> TokenToId(std::string_view token) const;
+ std::optional<std::string> IdToToken(int32_t id) const;
+ std::optional<int32_t> SpecialId(std::string_view content) const;
+ std::map<std::string, int32_t> SpecialTokens() const;
+ std::optional<int32_t> Cls() const;
+ std::optional<int32_t> Sep() const;
+ std::optional<int32_t> Mask() const;
+ std::optional<int32_t> Pad() const;
+ std::string MaskToken() const; // "" when none is configured
+
+ // Test seams: the stages of Encode, so a stage can be compared with the reference on its own.
+ std::string NormalizeForTest(std::string_view text) const;
+ std::vector<std::string> PreTokenizeForTest(std::string_view normalized) const;
+
+ private:
+ explicit Tokenizer(std::shared_ptr<const Impl> impl) : impl_(std::move(impl)) {}
+ std::shared_ptr<const Impl> impl_;
+};
+
+// Stages exposed for the golden tests that sweep them without a tokenizer.json.
+enum class Form { NFC, NFD, NFKC, NFKD };
+std::string Normalize(std::string_view text, Form form);
+std::string Lowercase(std::string_view text); // the crate's, no final-sigma rule
+
+// A normalizer or pre-tokenizer built from its tokenizer.json section, e.g. {"type":"Lowercase"}.
+Result<std::string> RunNormalizerSpec(std::string_view spec_json, std::string_view input);
+Result<std::vector<std::string>> RunPreTokenizerSpec(std::string_view spec_json, std::string_view input);
+
+} // namespace tokenizer
+} // namespace laya
diff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cpp
new file mode 100644
index 00000000..47443fa1
--- /dev/null
+++ b/laya-cpp/src/tokenizer.cpp
@@ -0,0 +1,1758 @@
+// Port of laya-go/tokenizer. Internal failures are a private exception caught at the API boundary
+// (Parse and the *Spec helpers); no exception crosses the public API.
+#include "laya/tokenizer.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <fstream>
+#include <functional>
+#include <memory>
+#include <queue>
+#include <sstream>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "laya/pyvalue.hpp"
+#include "laya/unicodedata.hpp"
+
+namespace laya {
+namespace tokenizer {
+
+const char* const kUnsupported = "unsupported tokenizer component: ";
+bool IsUnsupported(const std::string& e) { return e.rfind(kUnsupported, 0) == 0; }
+
+namespace {
+#include "tokenizer_tables.inc"
+
+using pyvalue::Value;
+namespace ud = unicodedata;
+
+struct Fail {
+ std::string msg;
+};
+[[noreturn]] void Refuse(const std::string& m) { throw Fail{m}; }
+[[noreturn]] void Unsupported(const std::string& m) { throw Fail{std::string(kUnsupported) + m}; }
+
+template <class T, size_t N>
+constexpr size_t Len(const T (&)[N]) {
+ return N;
+}
+
+// ------------------------------------------------------------------------------------------
+// Character classes: the reference's, from the generated tables.
+// ------------------------------------------------------------------------------------------
+bool IsWhiteSpace(char32_t r) {
+ switch (r) {
+ case '\t': case '\n': case 0x0B: case '\f': case '\r': case ' ': case 0x85: case 0xA0:
+ case 0x1680: case 0x2028: case 0x2029: case 0x202F: case 0x205F: case 0x3000:
+ return true;
+ }
+ return r >= 0x2000 && r <= 0x200A;
+}
+
+template <size_t N>
+bool InRanges(const uint32_t (&pairs)[N], char32_t r) {
+ long lo = 0, hi = long(N / 2) - 1;
+ while (lo <= hi) {
+ long mid = (lo + hi) / 2;
+ if (r < pairs[mid * 2]) hi = mid - 1;
+ else if (r > pairs[mid * 2 + 1]) lo = mid + 1;
+ else return true;
+ }
+ return false;
+}
+bool IsLetter(char32_t r) {
+ if (r < 0x80) return 'a' <= (r | 0x20) && (r | 0x20) <= 'z';
+ return InRanges(k_letter, r);
+}
+bool IsNumber(char32_t r) {
+ if (r < 0x80) return '0' <= r && r <= '9';
+ return InRanges(k_number, r);
+}
+bool IsWordChar(char32_t r) { return InRanges(k_word_char, r); }
+
+// ------------------------------------------------------------------------------------------
+// Lowercase (per character, no final sigma) and normalization (UAX #15) from the crate's tables.
+// ------------------------------------------------------------------------------------------
+void LowerOf(char32_t r, std::string& out) {
+ if (r < 0x80) {
+ out.push_back(char((r >= 'A' && r <= 'Z') ? r + 32 : r));
+ return;
+ }
+ for (const auto& m : k_lower_multi) {
+ if (m.from == r) {
+ for (uint32_t i = 0; i < m.count; i++) ud::AppendRune(out, m.to[i]);
+ return;
+ }
+ }
+ const uint32_t* end = k_lower_from + Len(k_lower_from);
+ const uint32_t* at = std::lower_bound(k_lower_from, end, uint32_t(r));
+ if (at != end && *at == r) {
+ ud::AppendRune(out, k_lower_to[at - k_lower_from]);
+ return;
+ }
+ ud::AppendRune(out, r);
+}
+
+std::vector<char32_t> Runes(std::string_view s) {
+ std::vector<char32_t> out;
+ out.reserve(s.size());
+ while (!s.empty()) {
+ auto d = ud::DecodeRune(s);
+ out.push_back(d.rune);
+ s.remove_prefix(d.width);
+ }
+ return out;
+}
+std::string FromRunes(const std::vector<char32_t>& rs) {
+ std::string out;
+ out.reserve(rs.size());
+ for (char32_t r : rs) ud::AppendRune(out, r);
+ return out;
+}
+
+constexpr char32_t kSBase = 0xAC00, kLBase = 0x1100, kVBase = 0x1161, kTBase = 0x11A7;
+constexpr char32_t kLCount = 19, kVCount = 21, kTCount = 28, kNCount = kVCount * kTCount,
+ kSCount = kLCount * kNCount;
+
+bool LookupPool(const uint32_t* keys, size_t nkeys, const uint32_t* offsets, const uint32_t* pool,
+ char32_t r, const uint32_t** begin, const uint32_t** end) {
+ const uint32_t* at = std::lower_bound(keys, keys + nkeys, uint32_t(r));
+ if (at == keys + nkeys || *at != r) return false;
+ size_t i = size_t(at - keys);
+ *begin = pool + offsets[i];
+ *end = pool + offsets[i + 1];
+ return true;
+}
+
+void Decompose(std::vector<char32_t>& dst, char32_t r, bool compat) {
+ if (r < 0xA0 || (r < 0xC0 && !compat)) {
+ dst.push_back(r);
+ return;
+ }
+ if (r >= kSBase && r < kSBase + kSCount) {
+ char32_t s = r - kSBase;
+ dst.push_back(kLBase + s / kNCount);
+ dst.push_back(kVBase + (s % kNCount) / kTCount);
+ if (s % kTCount != 0) dst.push_back(kTBase + s % kTCount);
+ return;
+ }
+ const uint32_t *b, *e;
+ if (compat && LookupPool(k_compat_keys, Len(k_compat_keys), k_compat_offsets, k_compat_pool, r, &b, &e)) {
+ dst.insert(dst.end(), b, e);
+ return;
+ }
+ if (LookupPool(k_canon_keys, Len(k_canon_keys), k_canon_offsets, k_canon_pool, r, &b, &e)) {
+ dst.insert(dst.end(), b, e);
+ return;
+ }
+ dst.push_back(r);
+}
+
+uint8_t CombiningClass(char32_t r) {
+ if (r < 0x300) return 0;
+ size_t n = Len(k_ccc_ranges) / 3, lo = 0, hi = n;
+ while (lo < hi) { // first triple whose hi >= r
+ size_t mid = (lo + hi) / 2;
+ if (k_ccc_ranges[mid * 3 + 1] >= r) hi = mid;
+ else lo = mid + 1;
+ }
+ if (lo < n && k_ccc_ranges[lo * 3] <= r) return uint8_t(k_ccc_ranges[lo * 3 + 2]);
+ return 0;
+}
+
+void Reorder(std::vector<char32_t>& rs) {
+ for (size_t i = 0; i < rs.size();) {
+ if (CombiningClass(rs[i]) == 0) {
+ i++;
+ continue;
+ }
+ size_t j = i;
+ while (j < rs.size() && CombiningClass(rs[j]) != 0) j++;
+ std::stable_sort(rs.begin() + long(i), rs.begin() + long(j),
+ [](char32_t a, char32_t b) { return CombiningClass(a) < CombiningClass(b); });
+ i = j;
+ }
+}
+
+bool ComposePair(char32_t first, char32_t second, char32_t* out) {
+ if (first >= kLBase && first < kLBase + kLCount && second >= kVBase && second < kVBase + kVCount) {
+ *out = kSBase + ((first - kLBase) * kVCount + (second - kVBase)) * kTCount;
+ return true;
+ }
+ if (first >= kSBase && first < kSBase + kSCount && (first - kSBase) % kTCount == 0 &&
+ second > kTBase && second < kTBase + kTCount) {
+ *out = first + (second - kTBase);
+ return true;
+ }
+ uint64_t key = (uint64_t(first) << 32) | second;
+ const uint64_t* end = k_compose_keys + Len(k_compose_keys);
+ const uint64_t* at = std::lower_bound(k_compose_keys, end, key);
+ if (at != end && *at == key) {
+ *out = k_compose_values[at - k_compose_keys];
+ return true;
+ }
+ return false;
+}
+
+void ComposeRunes(std::vector<char32_t>& rs) {
+ std::vector<char32_t> out;
+ out.reserve(rs.size());
+ long starter = -1;
+ uint8_t last = 0;
+ for (char32_t r : rs) {
+ uint8_t cc = CombiningClass(r);
+ char32_t c;
+ if (starter >= 0 && (last < cc || last == 0) && ComposePair(out[size_t(starter)], r, &c)) {
+ out[size_t(starter)] = c;
+ continue;
+ }
+ if (cc == 0) {
+ starter = long(out.size());
+ last = 0;
+ } else {
+ last = cc;
+ }
+ out.push_back(r);
+ }
+ rs.swap(out);
+}
+
+std::string NormalizeImpl(std::string_view s, Form f) {
+ bool compat = f == Form::NFKC || f == Form::NFKD, compose = f == Form::NFC || f == Form::NFKC;
+ bool ascii = true, below = true;
+ for (char32_t r : Runes(s)) {
+ if (r >= 0x80) ascii = false;
+ if (r >= 0x300) {
+ below = false;
+ break;
+ }
+ }
+ // ASCII has no decomposition in any form, and nothing below U+0300 combines with what precedes.
+ if (ascii || (below && compose && !compat)) return std::string(s);
+ std::vector<char32_t> rs;
+ for (char32_t r : Runes(s)) Decompose(rs, r, compat);
+ Reorder(rs);
+ if (compose) ComposeRunes(rs);
+ return FromRunes(rs);
+}
+
+std::string LowercaseImpl(std::string_view s) {
+ std::string out;
+ out.reserve(s.size());
+ while (!s.empty()) {
+ auto d = ud::DecodeRune(s);
+ LowerOf(d.rune, out);
+ s.remove_prefix(d.width);
+ }
+ return out;
+}
+
+std::string StripImpl(std::string_view s, bool left, bool right) {
+ size_t a = 0, b = s.size();
+ if (left) {
+ while (a < b) {
+ auto d = ud::DecodeRune(s.substr(a, b - a));
+ if (!IsWhiteSpace(d.rune)) break;
+ a += d.width;
+ }
+ }
+ if (right) {
+ while (b > a) {
+ auto d = ud::DecodeLastRune(s.substr(a, b - a));
+ if (!IsWhiteSpace(d.rune)) break;
+ b -= d.width;
+ }
+ }
+ return std::string(s.substr(a, b - a));
+}
+
+// ------------------------------------------------------------------------------------------
+// A small regex for the Replace normalizer. The reference reads \s \d \w \b \p as Unicode, and a
+// stand-in regex engine would read them its own way, so those are refused (as laya-go does); what
+// is left is literals, classes of explicit ranges, ., ^, $, groups, | and the greedy and lazy
+// repeats. Matching is a backtracker that remembers failed (pc, position) pairs, so it is linear.
+// ------------------------------------------------------------------------------------------
+class Regex {
+ public:
+ explicit Regex(const std::string& pattern) { Compile(pattern); }
+
+ // Replaces every non-overlapping match with the literal content.
+ std::string ReplaceAll(const std::string& text, const std::string& content) const {
+ std::string out;
+ size_t copied = 0, from = 0;
+ while (from <= text.size()) {
+ size_t ms, me;
+ if (!Find(text, from, &ms, &me)) break;
+ out.append(text, copied, ms - copied);
+ out += content;
+ copied = me;
+ from = me; // matches are never empty: Compile refuses a pattern that can match nothing
+ }
+ out.append(text, copied, std::string::npos);
+ return out;
+ }
+ bool MatchesEmpty() const {
+ size_t s, e;
+ return Find("", 0, &s, &e);
+ }
+
+ private:
+ enum Op { kChar, kAny, kSet, kSplit, kJmp, kBol, kEol, kMatch };
+ struct Inst {
+ Op op;
+ char32_t c = 0;
+ int x = 0, y = 0;
+ };
+ struct Set {
+ std::vector<std::pair<char32_t, char32_t>> ranges;
+ bool negate = false;
+ };
+ std::vector<Inst> prog_;
+ std::vector<Set> sets_;
+
+ // --- parser: pattern -> instructions, emitted directly from a recursive descent -------------
+ std::vector<char32_t> p_;
+ size_t at_ = 0;
+ bool icase_ = false;
+
+ bool More() const { return at_ < p_.size(); }
+ char32_t Peek() const { return p_[at_]; }
+ int Emit(Op op, char32_t c = 0, int x = 0, int y = 0) {
+ prog_.push_back(Inst{op, c, x, y});
+ return int(prog_.size()) - 1;
+ }
+
+ void Compile(const std::string& pattern) {
+ p_ = Runes(pattern);
+ if (p_.size() >= 4 && p_[0] == '(' && p_[1] == '?' && p_[2] == 'i' && p_[3] == ')') {
+ for (char32_t r : p_) {
+ if (r >= 0x80) Unsupported("a case-insensitive Replace regex with a non-ASCII character is not implemented");
+ }
+ icase_ = true;
+ at_ = 4;
+ }
+ Alternation();
+ if (More()) Refuse("a Replace normalizer's regex: unexpected " + std::string(1, char(Peek())));
+ Emit(kMatch);
+ if (MatchesEmpty()) {
+ Unsupported("this Replace regex can match the empty string, whose replacement rules differ between engines");
+ }
+ }
+
+ void Alternation() {
+ int start = int(prog_.size());
+ Concatenation();
+ std::vector<int> jumps;
+ while (More() && Peek() == '|') {
+ at_++;
+ // split(start of the left branches, next), the left branch jumps over the right.
+ prog_.insert(prog_.begin() + start, Inst{kSplit, 0, 0, 0});
+ Shift(start + 1, 1);
+ int j = Emit(kJmp);
+ jumps.push_back(j);
+ prog_[size_t(start)].x = start + 1;
+ prog_[size_t(start)].y = int(prog_.size());
+ Concatenation();
+ }
+ for (int j : jumps) prog_[size_t(j)].x = int(prog_.size());
+ }
+
+ // Inserting an instruction at `from` moves everything after it, so targets at or past `from`
+ // that were emitted before the insertion move by `by`.
+ void Shift(int from, int by) {
+ for (size_t i = size_t(from); i < prog_.size(); i++) {
+ Inst& in = prog_[i];
+ if (in.op == kSplit || in.op == kJmp) {
+ if (in.x >= from) in.x += by;
+ if (in.op == kSplit && in.y >= from) in.y += by;
+ }
+ }
+ }
+
+ void Concatenation() {
+ while (More() && Peek() != '|' && Peek() != ')') Repeat();
+ }
+
+ void Repeat() {
+ int start = int(prog_.size());
+ Atom();
+ int end = int(prog_.size());
+ while (More()) {
+ char32_t c = Peek();
+ int lo, hi; // hi < 0: unbounded
+ if (c == '*') { lo = 0; hi = -1; at_++; }
+ else if (c == '+') { lo = 1; hi = -1; at_++; }
+ else if (c == '?') { lo = 0; hi = 1; at_++; }
+ else if (c == '{') {
+ size_t save = at_;
+ if (!Counts(&lo, &hi)) { at_ = save; break; }
+ } else break;
+ bool lazy = More() && Peek() == '?';
+ if (lazy) at_++;
+ std::vector<Inst> body(prog_.begin() + start, prog_.begin() + end);
+ for (Inst& in : body) {
+ if (in.op == kSplit || in.op == kJmp) {
+ in.x -= start;
+ if (in.op == kSplit) in.y -= start;
+ }
+ }
+ prog_.resize(size_t(start));
+ if (lo > 1000 || hi > 1000) Refuse("a Replace regex repeat count above 1000");
+ auto copy = [&]() {
+ int base = int(prog_.size());
+ for (Inst in : body) {
+ if (in.op == kSplit || in.op == kJmp) {
+ in.x += base;
+ if (in.op == kSplit) in.y += base;
+ }
+ prog_.push_back(in);
+ }
+ };
+ for (int i = 0; i < lo; i++) copy();
+ if (hi < 0) { // L: split(body, out); body; jmp L
+ int l = int(prog_.size());
+ int s = Emit(kSplit);
+ copy();
+ Emit(kJmp, 0, l);
+ int out = int(prog_.size());
+ prog_[size_t(s)].x = lazy ? out : s + 1;
+ prog_[size_t(s)].y = lazy ? s + 1 : out;
+ } else {
+ std::vector<int> splits;
+ for (int i = lo; i < hi; i++) {
+ splits.push_back(Emit(kSplit));
+ copy();
+ }
+ int out = int(prog_.size());
+ for (int s : splits) {
+ prog_[size_t(s)].x = lazy ? out : s + 1;
+ prog_[size_t(s)].y = lazy ? s + 1 : out;
+ }
+ }
+ end = int(prog_.size());
+ }
+ }
+
+ bool Counts(int* lo, int* hi) { // at '{'; false when it is not a repeat count (then a literal)
+ at_++;
+ auto number = [&](int* v) {
+ if (!More() || Peek() < '0' || Peek() > '9') return false;
+ long n = 0;
+ while (More() && Peek() >= '0' && Peek() <= '9') {
+ n = n * 10 + long(Peek() - '0');
+ if (n > 100000) n = 100000;
+ at_++;
+ }
+ *v = int(n);
+ return true;
+ };
+ if (!number(lo)) return false;
+ *hi = *lo;
+ if (More() && Peek() == ',') {
+ at_++;
+ if (More() && Peek() == '}') *hi = -1;
+ else if (!number(hi)) return false;
+ }
+ if (!More() || Peek() != '}') return false;
+ at_++;
+ if (*hi >= 0 && *hi < *lo) Refuse("a Replace regex repeat count with its bounds reversed");
+ return true;
+ }
+
+ void Atom() {
+ char32_t c = Peek();
+ if (c == '(') {
+ at_++;
+ if (More() && Peek() == '?') {
+ if (at_ + 1 < p_.size() && p_[at_ + 1] == ':') at_ += 2;
+ else Unsupported("this Replace regex group syntax (?...) is not implemented");
+ }
+ Alternation();
+ if (!More() || Peek() != ')') Refuse("a Replace normalizer's regex: missing )");
+ at_++;
+ } else if (c == '[') {
+ Class();
+ } else if (c == '.') {
+ at_++;
+ Emit(kAny);
+ } else if (c == '^') {
+ at_++;
+ Emit(kBol);
+ } else if (c == '$') {
+ at_++;
+ Emit(kEol);
+ } else if (c == '*' || c == '+' || c == '?') {
+ Refuse("a Replace normalizer's regex: nothing to repeat");
+ } else if (c == '\\') {
+ Literal(Escape());
+ } else {
+ at_++;
+ Literal(c);
+ }
+ }
+
+ char32_t Escape() { // at '\\'
+ at_++;
+ if (!More()) Refuse("a Replace normalizer's regex ends in a backslash");
+ char32_t c = Peek();
+ at_++;
+ switch (c) {
+ case 't': return '\t';
+ case 'n': return '\n';
+ case 'r': return '\r';
+ case 'f': return '\f';
+ case 'v': return 0x0B;
+ default: break;
+ }
+ if (c < 0x80 && !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'))) return c;
+ Unsupported("the Replace regex escape \\" + std::string(1, char(c < 0x80 ? c : '?')) + " is not implemented");
+ }
+
+ void Literal(char32_t c) {
+ if (icase_ && ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) {
+ Set s;
+ s.ranges = {{c | 0x20, c | 0x20}, {c & ~char32_t(0x20), c & ~char32_t(0x20)}};
+ sets_.push_back(s);
+ Emit(kSet, 0, int(sets_.size()) - 1);
+ } else {
+ Emit(kChar, c);
+ }
+ }
+
+ void Class() { // at '['
+ at_++;
+ Set s;
+ if (More() && Peek() == '^') {
+ s.negate = true;
+ at_++;
+ }
+ bool first = true;
+ while (true) {
+ if (!More()) Refuse("a Replace normalizer's regex: missing ]");
+ char32_t c = Peek();
+ if (c == ']' && !first) { at_++; break; }
+ first = false;
+ if (c == '[') Unsupported("a nested [ or a POSIX class in a Replace regex is not implemented");
+ char32_t lo = c == '\\' ? Escape() : (at_++, c);
+ char32_t hi = lo;
+ if (More() && Peek() == '-' && at_ + 1 < p_.size() && p_[at_ + 1] != ']') {
+ at_++;
+ char32_t h = Peek();
+ if (h == '[') Unsupported("a nested [ in a Replace regex is not implemented");
+ hi = h == '\\' ? Escape() : (at_++, h);
+ if (hi < lo) Refuse("a Replace normalizer's regex: a range with its ends reversed");
+ }
+ s.ranges.push_back({lo, hi});
+ }
+ if (icase_) {
+ auto more = s.ranges;
+ for (auto& r : s.ranges) {
+ char32_t a = std::max<char32_t>(r.first, 'a'), b = std::min<char32_t>(r.second, 'z');
+ if (a <= b) more.push_back({a & ~char32_t(0x20), b & ~char32_t(0x20)});
+ a = std::max<char32_t>(r.first, 'A');
+ b = std::min<char32_t>(r.second, 'Z');
+ if (a <= b) more.push_back({a | 0x20, b | 0x20});
+ }
+ s.ranges = more;
+ }
+ sets_.push_back(s);
+ Emit(kSet, 0, int(sets_.size()) - 1);
+ }
+
+ // --- matcher --------------------------------------------------------------------------------
+ bool Find(const std::string& text, size_t from, size_t* ms, size_t* me) const {
+ size_t width = text.size() + 1;
+ std::vector<bool> seen(prog_.size() * width, false);
+ std::vector<std::pair<int, size_t>> stack;
+ for (size_t start = from; start <= text.size();) {
+ stack.assign(1, {0, start});
+ while (!stack.empty()) {
+ auto [pc, pos] = stack.back();
+ stack.pop_back();
+ while (true) {
+ size_t key = size_t(pc) * width + pos;
+ if (seen[key]) break;
+ seen[key] = true;
+ const Inst& in = prog_[size_t(pc)];
+ if (in.op == kMatch) {
+ *ms = start;
+ *me = pos;
+ return true;
+ }
+ if (in.op == kJmp) { pc = in.x; continue; }
+ if (in.op == kSplit) {
+ stack.push_back({in.y, pos});
+ pc = in.x;
+ continue;
+ }
+ if (in.op == kBol) { if (pos != 0) break; pc++; continue; }
+ if (in.op == kEol) { if (pos != text.size()) break; pc++; continue; }
+ if (pos >= text.size()) break;
+ auto d = ud::DecodeRune(std::string_view(text).substr(pos));
+ bool ok;
+ if (in.op == kChar) ok = d.rune == in.c;
+ else if (in.op == kAny) ok = d.rune != '\n';
+ else {
+ const Set& s = sets_[size_t(in.x)];
+ bool hit = false;
+ for (auto& r : s.ranges) hit = hit || (d.rune >= r.first && d.rune <= r.second);
+ ok = hit != s.negate;
+ }
+ if (!ok) break;
+ pos += d.width;
+ pc++;
+ }
+ }
+ start += start < text.size() ? ud::DecodeRune(std::string_view(text).substr(start)).width : 1;
+ }
+ return false;
+ }
+};
+
+// ------------------------------------------------------------------------------------------
+// Component sections of tokenizer.json.
+// ------------------------------------------------------------------------------------------
+using Normalizer = std::function<std::string(const std::string&)>;
+
+const Value* Member(const Value& node, const char* key) {
+ const Value* v = node.Find(key);
+ return (v == nullptr || v->is_null()) ? nullptr : v;
+}
+std::string Str(const Value& n, const char* key, const std::string& def) {
+ const Value* v = Member(n, key);
+ if (v == nullptr) return def;
+ if (v->kind() != Value::Kind::String) Refuse(std::string("\"") + key + "\" must be a string, found " + pyvalue::TypeName(*v));
+ return v->as_string();
+}
+bool Bool(const Value& n, const char* key, bool def) {
+ const Value* v = Member(n, key);
+ if (v == nullptr) return def;
+ if (v->kind() != Value::Kind::Bool) Refuse(std::string("\"") + key + "\" must be a boolean, found " + pyvalue::TypeName(*v));
+ return v->as_bool();
+}
+int Int(const Value& n, const char* key, int def) {
+ const Value* v = Member(n, key);
+ if (v == nullptr) return def;
+ if (v->kind() == Value::Kind::Int && v->as_int() >= -1000000000 && v->as_int() <= 1000000000) return int(v->as_int());
+ if (v->kind() == Value::Kind::Float && v->as_float() == double(int(v->as_float()))) return int(v->as_float());
+ Refuse(std::string("\"") + key + "\" must be an integer");
+}
+const Value& Object(const Value& n, const char* key) {
+ const Value* v = n.Find(key);
+ if (v == nullptr || v->kind() != Value::Kind::Object) Refuse(std::string("\"") + key + "\" must be an object");
+ return *v;
+}
+const Value& Array(const Value& n, const char* key) {
+ const Value* v = n.Find(key);
+ if (v == nullptr || v->kind() != Value::Kind::Array) Refuse(std::string("\"") + key + "\" must be an array");
+ return *v;
+}
+std::string TypeOf(const Value& n, const char* what) {
+ if (n.kind() != Value::Kind::Object) Refuse(std::string(what) + " must be an object or null");
+ std::string kind = Str(n, "type", "");
+ if (kind.empty()) Refuse(std::string("a ") + what + " must declare a type");
+ return kind;
+}
+
+bool DialectDiffers(const std::string& expr, char* which) {
+ for (size_t i = 0; i + 1 < expr.size(); i++) {
+ if (expr[i] != '\\') continue;
+ i++;
+ if (std::string("sSdDwWbBpP").find(expr[i]) != std::string::npos) {
+ *which = expr[i];
+ return true;
+ }
+ }
+ return false;
+}
+
+std::string ReplaceAllLiteral(const std::string& s, const std::string& from, const std::string& to) {
+ std::string out;
+ size_t at = 0;
+ while (true) {
+ size_t hit = s.find(from, at);
+ if (hit == std::string::npos) break;
+ out.append(s, at, hit - at);
+ out += to;
+ at = hit + from.size();
+ }
+ out.append(s, at, std::string::npos);
+ return out;
+}
+
+Normalizer ParseNormalizer(const Value* n) {
+ if (n == nullptr || n->is_null()) return [](const std::string& s) { return s; };
+ std::string kind = TypeOf(*n, "normalizer");
+ if (kind == "NFC") return [](const std::string& s) { return NormalizeImpl(s, Form::NFC); };
+ if (kind == "NFD") return [](const std::string& s) { return NormalizeImpl(s, Form::NFD); };
+ if (kind == "NFKC") return [](const std::string& s) { return NormalizeImpl(s, Form::NFKC); };
+ if (kind == "NFKD") return [](const std::string& s) { return NormalizeImpl(s, Form::NFKD); };
+ if (kind == "Lowercase") return [](const std::string& s) { return LowercaseImpl(s); };
+ if (kind == "Prepend") {
+ std::string prefix = Str(*n, "prepend", "");
+ return [prefix](const std::string& s) { return s.empty() ? s : prefix + s; }; // non-empty text only
+ }
+ if (kind == "Strip") {
+ bool l = Bool(*n, "strip_left", true), r = Bool(*n, "strip_right", true);
+ return [l, r](const std::string& s) { return StripImpl(s, l, r); };
+ }
+ if (kind == "Replace") {
+ const Value& pattern = Object(*n, "pattern");
+ const Value* content = n->Find("content");
+ if (content == nullptr || content->kind() != Value::Kind::String) Refuse("a Replace normalizer must declare content");
+ std::string to = content->as_string();
+ if (const Value* lit = pattern.Find("String"); lit != nullptr && lit->kind() == Value::Kind::String) {
+ std::string from = lit->as_string();
+ if (from.empty()) Refuse("a Replace normalizer cannot match the empty string");
+ return [from, to](const std::string& s) { return ReplaceAllLiteral(s, from, to); };
+ }
+ const Value* rx = pattern.Find("Regex");
+ if (rx == nullptr || rx->kind() != Value::Kind::String) {
+ Refuse("a Replace normalizer's pattern must be {\"String\": ...} or {\"Regex\": ...}");
+ }
+ char which;
+ if (DialectDiffers(rx->as_string(), &which)) {
+ Unsupported(std::string("the Replace regex uses \\") + which +
+ ", whose meaning differs between the reference's regex engine (Unicode-aware) and "
+ "an ASCII one, so it would normalize differently; refusing to load");
+ }
+ auto re = std::make_shared<Regex>(rx->as_string());
+ return [re, to](const std::string& s) { return re->ReplaceAll(s, to); };
+ }
+ if (kind == "Sequence") {
+ std::vector<Normalizer> steps;
+ for (const Value& child : Array(*n, "normalizers").items()) {
+ if (child.kind() != Value::Kind::Object) Refuse("a normalizer in a Sequence must be an object");
+ steps.push_back(ParseNormalizer(&child));
+ }
+ return [steps](const std::string& s) {
+ std::string cur = s;
+ for (const auto& step : steps) cur = step(cur);
+ return cur;
+ };
+ }
+ Unsupported("the normalizer \"" + kind + "\" is not implemented; refusing to load rather than produce ids "
+ "that are wrong in a way no answer-level test would localise");
+}
+
+// The byte-level stand-in alphabet: printable ASCII and Latin-1 bytes map to themselves, the other
+// 68 to U+0100 upward.
+struct Alphabet {
+ char32_t to_rune[256];
+ int16_t to_byte[0x144];
+ Alphabet() {
+ std::fill(std::begin(to_byte), std::end(to_byte), int16_t(-1));
+ int lifted = 0;
+ for (int b = 0; b < 256; b++) {
+ if ((b >= '!' && b <= '~') || (b >= 0xA1 && b <= 0xAC) || (b >= 0xAE && b <= 0xFF)) to_rune[b] = char32_t(b);
+ else to_rune[b] = char32_t(256 + lifted++);
+ to_byte[to_rune[b]] = int16_t(b);
+ }
+ }
+ int ByteFor(char32_t r) const { return r >= 0x144 ? -1 : to_byte[r]; }
+ std::string Encode(std::string_view piece) const {
+ std::string out;
+ out.reserve(piece.size() + piece.size() / 4);
+ for (unsigned char c : piece) ud::AppendRune(out, to_rune[c]);
+ return out;
+ }
+};
+const Alphabet& TheAlphabet() {
+ static const Alphabet a;
+ return a;
+}
+
+class PreTokenizer {
+ public:
+ virtual ~PreTokenizer() = default;
+ virtual void Split(std::string_view text, std::vector<std::string>& out) const = 0;
+};
+
+class WholeText : public PreTokenizer {
+ public:
+ void Split(std::string_view text, std::vector<std::string>& out) const override {
+ if (!text.empty()) out.emplace_back(text);
+ }
+};
+
+class ByteLevel : public PreTokenizer {
+ public:
+ ByteLevel(bool prefix, bool regex) : add_prefix_space_(prefix), use_regex_(regex) {}
+ void Split(std::string_view text, std::vector<std::string>& out) const override {
+ if (text.empty()) return;
+ std::string source(text);
+ if (add_prefix_space_ && source[0] != ' ') source = " " + source;
+ if (!use_regex_) {
+ out.push_back(TheAlphabet().Encode(source));
+ return;
+ }
+ for (size_t at = 0; at < source.size();) {
+ size_t end = Cut(source, at);
+ out.push_back(TheAlphabet().Encode(std::string_view(source).substr(at, end - at)));
+ at = end;
+ }
+ }
+
+ private:
+ // The GPT-2 pattern 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
+ // as a scanner: the pattern has a lookahead, and no standard regex has it with Unicode classes.
+ static bool InClass(char32_t r, int branch) {
+ if (branch == 0) return IsLetter(r);
+ if (branch == 1) return IsNumber(r);
+ return !IsWhiteSpace(r) && !IsLetter(r) && !IsNumber(r);
+ }
+ static size_t RunEnd(const std::string& s, size_t at, int branch) {
+ while (at < s.size()) {
+ auto d = ud::DecodeRune(std::string_view(s).substr(at));
+ if (!InClass(d.rune, branch)) break;
+ at += d.width;
+ }
+ return at;
+ }
+ static size_t Cut(const std::string& s, size_t from) {
+ static const char* const kContractions[] = {"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"};
+ std::string_view rest = std::string_view(s).substr(from);
+ for (const char* c : kContractions) {
+ std::string_view cv(c);
+ if (rest.substr(0, cv.size()) == cv) return from + cv.size();
+ }
+ auto first = ud::DecodeRune(rest);
+ for (int branch = 0; branch < 3; branch++) {
+ if (first.rune == ' ') {
+ size_t after = from + 1;
+ if (after < s.size() && InClass(ud::DecodeRune(std::string_view(s).substr(after)).rune, branch)) {
+ return RunEnd(s, after, branch);
+ }
+ } else if (InClass(first.rune, branch)) {
+ return RunEnd(s, from, branch);
+ }
+ }
+ if (IsWhiteSpace(first.rune)) {
+ size_t end = from;
+ while (end < s.size()) {
+ auto d = ud::DecodeRune(std::string_view(s).substr(end));
+ if (!IsWhiteSpace(d.rune)) break;
+ end += d.width;
+ }
+ if (end >= s.size()) return end;
+ // \s+(?!\S) gives the last white-space character back to the next piece.
+ size_t w = ud::DecodeLastRune(std::string_view(s).substr(0, end)).width;
+ if (end - w > from) return end - w;
+ return end; // a single space: \s+(?!\S) fails and \s+ takes it
+ }
+ return from + first.width;
+ }
+ bool add_prefix_space_, use_regex_;
+};
+
+class Metaspace : public PreTokenizer {
+ public:
+ Metaspace(std::string replacement, bool prepend, bool split)
+ : replacement_(std::move(replacement)), prepend_(prepend), split_(split) {}
+ void Split(std::string_view text, std::vector<std::string>& out) const override {
+ if (text.empty()) return;
+ std::string source = ReplaceAllLiteral(std::string(text), " ", replacement_);
+ if (prepend_ && source.compare(0, replacement_.size(), replacement_) != 0) source = replacement_ + source;
+ if (!split_) {
+ out.push_back(source);
+ return;
+ }
+ size_t start = 0;
+ size_t from = ud::DecodeRune(source).width; // a marker at the very start does not split
+ while (true) {
+ size_t next = source.find(replacement_, from);
+ if (next == std::string::npos) break;
+ out.push_back(source.substr(start, next - start));
+ start = next;
+ from = next + replacement_.size();
+ }
+ if (start < source.size()) out.push_back(source.substr(start));
+ }
+
+ private:
+ std::string replacement_;
+ bool prepend_, split_;
+};
+
+class SequencePre : public PreTokenizer {
+ public:
+ std::vector<std::unique_ptr<PreTokenizer>> steps;
+ void Split(std::string_view text, std::vector<std::string>& out) const override {
+ std::vector<std::string> current{std::string(text)};
+ for (const auto& step : steps) {
+ std::vector<std::string> next;
+ for (const auto& piece : current) step->Split(piece, next);
+ current.swap(next);
+ }
+ out.insert(out.end(), current.begin(), current.end());
+ }
+};
+
+std::unique_ptr<PreTokenizer> ParsePreTokenizer(const Value* n) {
+ if (n == nullptr || n->is_null()) return std::make_unique<WholeText>();
+ std::string kind = TypeOf(*n, "pre_tokenizer");
+ if (kind == "ByteLevel") return std::make_unique<ByteLevel>(Bool(*n, "add_prefix_space", true), Bool(*n, "use_regex", true));
+ if (kind == "Metaspace") {
+ std::string replacement = Str(*n, "replacement", "\xE2\x96\x81");
+ if (ud::RuneCount(replacement) != 1) Refuse("a Metaspace replacement must be exactly one code point");
+ std::string scheme = Str(*n, "prepend_scheme", "always");
+ bool split = Bool(*n, "split", true);
+ if (scheme == "always" || scheme == "first") return std::make_unique<Metaspace>(replacement, true, split);
+ if (scheme == "never") return std::make_unique<Metaspace>(replacement, false, split);
+ Refuse("unknown Metaspace prepend_scheme \"" + scheme + "\"");
+ }
+ if (kind == "Sequence") {
+ auto seq = std::make_unique<SequencePre>();
+ for (const Value& child : Array(*n, "pretokenizers").items()) {
+ if (child.kind() != Value::Kind::Object) Refuse("a pre-tokenizer in a Sequence must be an object");
+ seq->steps.push_back(ParsePreTokenizer(&child));
+ }
+ return seq;
+ }
+ Unsupported("the pre-tokenizer \"" + kind + "\" is not implemented; refusing to load rather than produce ids "
+ "that are wrong in a way no answer-level test would localise");
+}
+
+// ------------------------------------------------------------------------------------------
+// Decoders.
+// ------------------------------------------------------------------------------------------
+class Decoder {
+ public:
+ virtual ~Decoder() = default;
+ virtual void Transform(std::vector<std::string>& tokens) const = 0;
+};
+
+class ByteLevelDecoder : public Decoder {
+ public:
+ // A token holding a character outside the alphabet is taken as plain text; the whole is then
+ // decoded as lossy UTF-8.
+ void Transform(std::vector<std::string>& tokens) const override {
+ std::string raw;
+ for (const auto& token : tokens) {
+ std::string mapped;
+ bool ok = true;
+ std::string_view rest(token);
+ while (!rest.empty()) {
+ auto d = ud::DecodeRune(rest);
+ int b = TheAlphabet().ByteFor(d.rune);
+ if (b < 0) {
+ ok = false;
+ break;
+ }
+ mapped.push_back(char(b));
+ rest.remove_prefix(d.width);
+ }
+ raw += ok ? mapped : token;
+ }
+ tokens.assign(1, ud::Lossy(raw));
+ }
+};
+
+class ReplaceDecoder : public Decoder {
+ public:
+ ReplaceDecoder(std::string p, std::string c) : pattern_(std::move(p)), content_(std::move(c)) {}
+ void Transform(std::vector<std::string>& tokens) const override {
+ for (auto& t : tokens) t = ReplaceAllLiteral(t, pattern_, content_);
+ }
+
+ private:
+ std::string pattern_, content_;
+};
+
+int HexValue(char c) {
+ if (c >= '0' && c <= '9') return c - '0';
+ if (c >= 'A' && c <= 'F') return c - 'A' + 10;
+ if (c >= 'a' && c <= 'f') return c - 'a' + 10;
+ return -1;
+}
+// "<0xNN>", read as Rust's from_str_radix does: lower-case digits, and a '+' on the high digit
+// leaves only the low digit.
+bool ByteToken(const std::string& t, unsigned char* out) {
+ if (t.size() != 6 || t[0] != '<' || t[1] != '0' || t[2] != 'x' || t[5] != '>') return false;
+ if (t[3] == '+') {
+ int l = HexValue(t[4]);
+ *out = (unsigned char)l;
+ return l >= 0;
+ }
+ int h = HexValue(t[3]), l = HexValue(t[4]);
+ if (h < 0 || l < 0) return false;
+ *out = (unsigned char)(h << 4 | l);
+ return true;
+}
+
+class ByteFallbackDecoder : public Decoder {
+ public:
+ void Transform(std::vector<std::string>& tokens) const override {
+ std::vector<std::string> out;
+ std::string run;
+ auto flush = [&]() {
+ if (run.empty()) return;
+ if (ud::ValidUtf8(run)) out.push_back(run);
+ else for (size_t i = 0; i < run.size(); i++) out.push_back("\xEF\xBF\xBD");
+ run.clear();
+ };
+ for (const auto& token : tokens) {
+ unsigned char b;
+ if (ByteToken(token, &b)) {
+ run.push_back(char(b));
+ continue;
+ }
+ flush();
+ out.push_back(token);
+ }
+ flush();
+ tokens.swap(out);
+ }
+};
+
+class FuseDecoder : public Decoder {
+ public:
+ void Transform(std::vector<std::string>& tokens) const override {
+ std::string all;
+ for (const auto& t : tokens) all += t;
+ tokens.assign(1, all);
+ }
+};
+
+class StripDecoder : public Decoder {
+ public:
+ StripDecoder(std::string c, int start, int stop) : c_(std::move(c)), start_(start), stop_(stop) {}
+ void Transform(std::vector<std::string>& tokens) const override {
+ for (auto& t : tokens) {
+ for (int n = 0; n < start_ && t.compare(0, c_.size(), c_) == 0; n++) t.erase(0, c_.size());
+ for (int n = 0; n < stop_ && t.size() >= c_.size() && t.compare(t.size() - c_.size(), c_.size(), c_) == 0; n++) {
+ t.erase(t.size() - c_.size());
+ }
+ }
+ }
+
+ private:
+ std::string c_;
+ int start_, stop_;
+};
+
+class SequenceDecoder : public Decoder {
+ public:
+ std::vector<std::unique_ptr<Decoder>> steps;
+ void Transform(std::vector<std::string>& tokens) const override {
+ for (const auto& s : steps) s->Transform(tokens);
+ }
+};
+
+std::unique_ptr<Decoder> ParseDecoder(const Value* n) {
+ if (n == nullptr || n->is_null()) return nullptr;
+ std::string kind = TypeOf(*n, "decoder");
+ if (kind == "ByteLevel") return std::make_unique<ByteLevelDecoder>();
+ if (kind == "ByteFallback") return std::make_unique<ByteFallbackDecoder>();
+ if (kind == "Fuse") return std::make_unique<FuseDecoder>();
+ if (kind == "Replace") {
+ const Value* p = n->Find("pattern");
+ std::string literal;
+ if (p != nullptr && p->kind() == Value::Kind::Object) {
+ const Value* s = p->Find("String");
+ if (s == nullptr || s->kind() != Value::Kind::String) {
+ if (p->Find("Regex") != nullptr) {
+ Unsupported("only a literal String pattern is implemented in a Replace decoder; this checkpoint declares a Regex one");
+ }
+ Refuse("a Replace decoder needs a String pattern");
+ }
+ literal = s->as_string();
+ } else if (p != nullptr && p->kind() == Value::Kind::String) {
+ literal = p->as_string();
+ } else {
+ Refuse("a Replace decoder needs a String pattern");
+ }
+ if (literal.empty()) Refuse("a Replace decoder cannot match the empty string");
+ return std::make_unique<ReplaceDecoder>(literal, Str(*n, "content", ""));
+ }
+ if (kind == "Strip") {
+ std::string c = Str(*n, "content", " ");
+ if (ud::RuneCount(c) != 1) Refuse("a Strip decoder's content must be one character");
+ return std::make_unique<StripDecoder>(c, Int(*n, "start", 0), Int(*n, "stop", 0));
+ }
+ if (kind == "Sequence") {
+ auto seq = std::make_unique<SequenceDecoder>();
+ const Value* steps = n->Find("decoders");
+ if (steps == nullptr || steps->kind() != Value::Kind::Array) Refuse("a Sequence decoder needs a decoders array");
+ for (const Value& step : steps->items()) {
+ if (step.kind() != Value::Kind::Object) Refuse("every Sequence decoder step must be an object");
+ seq->steps.push_back(ParseDecoder(&step));
+ }
+ return seq;
+ }
+ Unsupported("the decoder \"" + kind + "\" is not implemented (ByteLevel, Replace, ByteFallback, Fuse, Strip and "
+ "Sequence are); refusing to load rather than decode it wrongly");
+}
+
+// ------------------------------------------------------------------------------------------
+// Added tokens.
+// ------------------------------------------------------------------------------------------
+struct AddedToken {
+ std::string content;
+ int32_t id;
+ bool special, lstrip, rstrip, single_word, normalized;
+};
+
+struct Span {
+ std::string text;
+ int32_t id = 0;
+ bool is_token = false;
+};
+
+// The added tokens that start with one character, longest first (the reference's leftmost-longest
+// match). `second` records which bytes follow that character in any of them, so the common miss is
+// rejected without trying each token.
+struct Bucket {
+ std::vector<const AddedToken*> tokens;
+ bool second[256] = {};
+ bool whole = false; // a token is that single character
+ void Add(const AddedToken* t, size_t first_width) {
+ tokens.push_back(t);
+ if (t->content.size() == first_width) whole = true;
+ else second[(unsigned char)t->content[first_width]] = true;
+ }
+};
+
+class AddedVocab {
+ public:
+ bool empty = true;
+ AddedVocab() = default;
+ explicit AddedVocab(const std::vector<AddedToken>& tokens) : empty(tokens.empty()), store_(tokens) {
+ for (const auto& t : store_) {
+ if (t.content.empty()) Refuse("an added token cannot have empty content");
+ auto d = ud::DecodeRune(t.content);
+ if (d.rune < 0x80) ascii_[d.rune].Add(&t, d.width);
+ else other_[d.rune].Add(&t, d.width);
+ }
+ auto longest_first = [](Bucket& b) {
+ std::sort(b.tokens.begin(), b.tokens.end(), [](const AddedToken* x, const AddedToken* y) {
+ if (x->content.size() != y->content.size()) return x->content.size() > y->content.size();
+ return x->content < y->content;
+ });
+ };
+ for (auto& b : ascii_) longest_first(b);
+ for (auto& kv : other_) longest_first(kv.second);
+ }
+
+ // Cuts text around its added tokens; lstrip and rstrip tokens absorb the white space beside them.
+ std::vector<Span> Split(const std::string& text) const {
+ std::vector<Span> out;
+ if (text.empty()) return out;
+ if (empty) {
+ out.push_back(Span{text, 0, false});
+ return out;
+ }
+ size_t plain_from = 0;
+ for (size_t at = 0; at < text.size();) {
+ auto d = ud::DecodeRune(std::string_view(text).substr(at));
+ const AddedToken* hit = LongestAt(text, at, d.rune, d.width);
+ if (hit == nullptr) {
+ at += d.width;
+ continue;
+ }
+ size_t start = at, stop = at + hit->content.size();
+ if (hit->lstrip) {
+ while (start > plain_from) {
+ auto p = ud::DecodeLastRune(std::string_view(text).substr(0, start));
+ if (!IsWhiteSpace(p.rune)) break;
+ start -= p.width;
+ }
+ }
+ if (hit->rstrip) {
+ while (stop < text.size()) {
+ auto nx = ud::DecodeRune(std::string_view(text).substr(stop));
+ if (!IsWhiteSpace(nx.rune)) break;
+ stop += nx.width;
+ }
+ }
+ if (start > plain_from) out.push_back(Span{text.substr(plain_from, start - plain_from), 0, false});
+ out.push_back(Span{hit->content, hit->id, true});
+ plain_from = at = stop;
+ }
+ if (plain_from < text.size()) out.push_back(Span{text.substr(plain_from), 0, false});
+ return out;
+ }
+
+ private:
+ const Bucket* BucketOf(char32_t r) const {
+ if (r < 0x80) return &ascii_[r];
+ auto it = other_.find(r);
+ return it == other_.end() ? nullptr : &it->second;
+ }
+ const AddedToken* LongestAt(const std::string& text, size_t at, char32_t first, size_t width) const {
+ const Bucket* b = BucketOf(first);
+ if (b == nullptr || b->tokens.empty()) return nullptr;
+ if (!b->whole) {
+ size_t next = at + width;
+ if (next >= text.size() || !b->second[(unsigned char)text[next]]) return nullptr;
+ }
+ for (const AddedToken* t : b->tokens) {
+ if (text.compare(at, t->content.size(), t->content) != 0) continue;
+ if (t->single_word && !StandsAlone(text, at, at + t->content.size())) continue;
+ return t;
+ }
+ return nullptr;
+ }
+ static bool StandsAlone(const std::string& text, size_t start, size_t stop) {
+ if (start > 0 && IsWordChar(ud::DecodeLastRune(std::string_view(text).substr(0, start)).rune)) return false;
+ if (stop < text.size() && IsWordChar(ud::DecodeRune(std::string_view(text).substr(stop)).rune)) return false;
+ return true;
+ }
+ std::vector<AddedToken> store_; // the buckets point into it; a moved vector keeps its buffer
+ Bucket ascii_[128];
+ std::unordered_map<char32_t, Bucket> other_;
+};
+
+// ------------------------------------------------------------------------------------------
+// BPE with byte fallback.
+// ------------------------------------------------------------------------------------------
+constexpr size_t kHeapMinSymbols = 32;
+constexpr int32_t kNoMerge = 1 << 30;
+constexpr char32_t kSmallRunes = 0x180;
+
+struct MergeRule {
+ int32_t rank = kNoMerge;
+ int32_t id = -1;
+};
+struct Symbol {
+ int32_t start, end, id; // id -1: a character the vocabulary lacks
+};
+
+uint64_t PairKey(int32_t l, int32_t r) { return (uint64_t(uint32_t(l)) << 32) | uint32_t(r); }
+
+class Bpe {
+ public:
+ Bpe(const std::unordered_map<std::string, int32_t>& vocab, const std::vector<std::pair<std::string, std::string>>& merges,
+ const std::string* unk, bool fuse_unk, bool ignore_merges, bool byte_fallback)
+ : vocab_(&vocab), fuse_unk_(fuse_unk), ignore_merges_(ignore_merges) {
+ for (char32_t r = 0; r < kSmallRunes; r++) {
+ std::string s;
+ ud::AppendRune(s, r);
+ auto it = vocab.find(s);
+ small_[r] = it == vocab.end() ? -1 : it->second;
+ }
+ if (unk != nullptr) {
+ auto it = vocab.find(*unk);
+ if (it == vocab.end()) {
+ Refuse("the tokenizer names " + *unk + " as its unknown token, but neither the vocabulary nor the added-token "
+ "table has an entry for it");
+ }
+ unk_id_ = it->second;
+ has_unk_ = true;
+ }
+ merges_.reserve(merges.size());
+ for (size_t rank = 0; rank < merges.size(); rank++) {
+ const auto& pair = merges[rank];
+ auto l = vocab.find(pair.first), r = vocab.find(pair.second);
+ if (l == vocab.end() || r == vocab.end()) {
+ Refuse("merge at rank " + std::to_string(rank) + " uses the symbol \"" + (l == vocab.end() ? pair.first : pair.second) +
+ "\", which the vocabulary does not contain; merging on ids would be unsound");
+ }
+ auto merged = vocab.find(pair.first + pair.second);
+ if (merged == vocab.end()) {
+ Refuse("merge at rank " + std::to_string(rank) + " produces \"" + pair.first + pair.second +
+ "\", which the vocabulary does not contain");
+ }
+ merges_.emplace(PairKey(l->second, r->second), MergeRule{int32_t(rank), merged->second}); // the lowest rank wins
+ }
+ if (byte_fallback) {
+ has_bytes_ = true;
+ for (int v = 0; v < 256; v++) {
+ char name[8];
+ std::snprintf(name, sizeof name, "<0x%02X>", v);
+ auto it = vocab.find(name);
+ byte_ids_[v] = it == vocab.end() ? -1 : it->second;
+ }
+ }
+ }
+
+ bool has_unk() const { return has_unk_; }
+ bool has_byte_fallback() const { return has_bytes_; }
+
+ void EncodePiece(const std::string& piece, std::vector<int32_t>& out) const {
+ if (piece.empty()) return;
+ if (ignore_merges_) {
+ auto it = vocab_->find(piece);
+ if (it != vocab_->end()) {
+ out.push_back(it->second);
+ return;
+ }
+ }
+ std::vector<Symbol> symbols;
+ symbols.reserve(piece.size());
+ for (size_t i = 0; i < piece.size();) {
+ auto d = ud::DecodeRune(std::string_view(piece).substr(i));
+ int32_t id;
+ if (d.rune < kSmallRunes && d.rune != ud::kReplacement) {
+ id = small_[d.rune];
+ } else {
+ auto it = vocab_->find(piece.substr(i, d.width));
+ id = it == vocab_->end() ? -1 : it->second;
+ }
+ symbols.push_back(Symbol{int32_t(i), int32_t(i + d.width), id});
+ i += d.width;
+ }
+ if (symbols.size() >= kHeapMinSymbols) MergeWithHeap(symbols);
+ else MergeByRescan(symbols);
+ Emit(piece, symbols, out);
+ }
+
+ // Both strategies are public for the test that they agree.
+ void MergeByRescan(std::vector<Symbol>& symbols) const {
+ std::vector<MergeRule> rules(symbols.size());
+ auto pair = [&](size_t i) { rules[i] = Rule(symbols[i], symbols[i + 1]); };
+ for (size_t i = 0; i + 1 < symbols.size(); i++) pair(i);
+ while (symbols.size() > 1) {
+ int32_t best_rank = kNoMerge;
+ long best_at = -1;
+ for (size_t i = 0; i + 1 < symbols.size(); i++) {
+ if (rules[i].rank < best_rank) {
+ best_rank = rules[i].rank;
+ best_at = long(i);
+ }
+ }
+ if (best_at < 0) return;
+ size_t b = size_t(best_at);
+ symbols[b] = Symbol{symbols[b].start, symbols[b + 1].end, rules[b].id};
+ symbols.erase(symbols.begin() + long(b) + 1);
+ rules.erase(rules.begin() + long(b) + 1);
+ if (b + 1 < symbols.size()) pair(b);
+ if (b > 0) pair(b - 1);
+ }
+ }
+
+ void MergeWithHeap(std::vector<Symbol>& symbols) const {
+ struct Item {
+ int32_t rank, slot, version;
+ bool operator>(const Item& o) const { return rank != o.rank ? rank > o.rank : slot > o.slot; }
+ };
+ size_t n = symbols.size();
+ std::vector<int32_t> next(n), prev(n), version(n, 0);
+ std::vector<char> dead(n, 0);
+ for (size_t i = 0; i < n; i++) {
+ prev[i] = int32_t(i) - 1;
+ next[i] = int32_t(i) + 1;
+ }
+ next[n - 1] = -1;
+ std::priority_queue<Item, std::vector<Item>, std::greater<Item>> queue;
+ auto offer = [&](int32_t slot) {
+ int32_t right = next[size_t(slot)];
+ if (right < 0) return;
+ MergeRule r = Rule(symbols[size_t(slot)], symbols[size_t(right)]);
+ if (r.rank != kNoMerge) queue.push(Item{r.rank, slot, version[size_t(slot)]});
+ };
+ for (size_t i = 0; i < n; i++) offer(int32_t(i));
+ while (!queue.empty()) {
+ Item item = queue.top();
+ queue.pop();
+ size_t slot = size_t(item.slot);
+ if (dead[slot] || version[slot] != item.version) continue;
+ int32_t right = next[slot];
+ if (right < 0 || dead[size_t(right)]) continue;
+ MergeRule r = Rule(symbols[slot], symbols[size_t(right)]);
+ if (r.rank == kNoMerge) continue;
+ symbols[slot] = Symbol{symbols[slot].start, symbols[size_t(right)].end, r.id};
+ dead[size_t(right)] = 1;
+ next[slot] = next[size_t(right)];
+ if (next[size_t(right)] >= 0) prev[size_t(next[size_t(right)])] = int32_t(slot);
+ version[slot]++;
+ offer(int32_t(slot));
+ if (prev[slot] >= 0) {
+ version[size_t(prev[slot])]++;
+ offer(prev[slot]);
+ }
+ }
+ std::vector<Symbol> out;
+ for (int32_t i = 0; i >= 0 && size_t(i) < n; i = next[size_t(i)]) {
+ if (!dead[size_t(i)]) out.push_back(symbols[size_t(i)]);
+ }
+ symbols.swap(out);
+ }
+
+ private:
+ MergeRule Rule(const Symbol& l, const Symbol& r) const {
+ if (l.id < 0 || r.id < 0) return MergeRule{};
+ auto it = merges_.find(PairKey(l.id, r.id));
+ return it == merges_.end() ? MergeRule{} : it->second;
+ }
+
+ // A symbol outside the vocabulary becomes its UTF-8 bytes when the checkpoint has byte fallback
+ // and every byte resolves, else the unknown token (one per run with fuse_unk), else it is dropped.
+ void Emit(const std::string& piece, const std::vector<Symbol>& symbols, std::vector<int32_t>& out) const {
+ bool unknown_pending = false;
+ for (const Symbol& s : symbols) {
+ if (s.id >= 0) {
+ unknown_pending = false;
+ out.push_back(s.id);
+ continue;
+ }
+ if (has_bytes_) {
+ bool resolves = true;
+ for (int32_t i = s.start; i < s.end; i++) resolves = resolves && byte_ids_[(unsigned char)piece[size_t(i)]] >= 0;
+ if (resolves) {
+ for (int32_t i = s.start; i < s.end; i++) out.push_back(byte_ids_[(unsigned char)piece[size_t(i)]]);
+ unknown_pending = false;
+ continue;
+ }
+ }
+ if (!has_unk_) continue;
+ if (fuse_unk_ && unknown_pending) continue;
+ out.push_back(unk_id_);
+ unknown_pending = true;
+ }
+ }
+
+ int32_t small_[kSmallRunes];
+ const std::unordered_map<std::string, int32_t>* vocab_;
+ std::unordered_map<uint64_t, MergeRule> merges_;
+ int32_t unk_id_ = 0;
+ bool has_unk_ = false, fuse_unk_, ignore_merges_, has_bytes_ = false;
+ int32_t byte_ids_[256];
+};
+
+} // namespace
+
+// ------------------------------------------------------------------------------------------
+// Tokenizer.
+// ------------------------------------------------------------------------------------------
+struct Impl {
+ Normalizer normalizer;
+ std::unique_ptr<PreTokenizer> pre;
+ std::unique_ptr<Decoder> decoder;
+ AddedVocab raw_added, normal_added;
+ std::unique_ptr<Bpe> bpe;
+ std::unordered_map<std::string, int32_t> ids;
+ std::vector<std::string> tokens;
+ std::map<std::string, int32_t> specials;
+ std::unordered_set<int32_t> is_special;
+ std::map<std::string, int32_t> roles;
+ std::string mask_token;
+ bool byte_level_only = false;
+};
+
+namespace {
+
+void ReadMerges(const Value& model, std::vector<std::pair<std::string, std::string>>& out) {
+ const Value* merges = model.Find("merges");
+ if (merges == nullptr || merges->is_null()) return;
+ if (merges->kind() != Value::Kind::Array) Refuse("model.merges must be an array");
+ out.reserve(merges->size());
+ for (const Value& m : merges->items()) {
+ if (m.kind() == Value::Kind::String) {
+ const std::string& j = m.as_string();
+ bool found = false;
+ for (size_t i = 1; i + 1 < j.size(); i++) {
+ if (j[i] == ' ') {
+ out.emplace_back(j.substr(0, i), j.substr(i + 1));
+ found = true;
+ break;
+ }
+ }
+ if (!found) Refuse("a space-joined merge must have two non-empty halves, got \"" + j + "\"");
+ } else if (m.kind() == Value::Kind::Array && m.size() == 2 && m.items()[0].kind() == Value::Kind::String &&
+ m.items()[1].kind() == Value::Kind::String) {
+ out.emplace_back(m.items()[0].as_string(), m.items()[1].as_string());
+ } else {
+ Refuse("a merge must be two strings or one space-joined string");
+ }
+ }
+}
+
+std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value* config) {
+ auto parsed = pyvalue::Parse(tokenizer_json);
+ if (!parsed.ok()) Refuse("tokenizer.json: " + parsed.error());
+ const Value& doc = parsed.value();
+ if (doc.kind() != Value::Kind::Object) Refuse("tokenizer.json must be a JSON object");
+ const Value* model_v = doc.Find("model");
+ if (model_v == nullptr || model_v->kind() != Value::Kind::Object) Refuse("model must be an object");
+ const Value& model = *model_v;
+ std::string type = Str(model, "type", "");
+ if (type != "BPE") {
+ Unsupported("this library implements the BPE tokenizer model; the checkpoint declares \"" + type +
+ "\". Refusing to load rather than mis-tokenize silently");
+ }
+ auto impl = std::make_shared<Impl>();
+ impl->normalizer = ParseNormalizer(Member(doc, "normalizer"));
+ impl->pre = ParsePreTokenizer(Member(doc, "pre_tokenizer"));
+ impl->decoder = ParseDecoder(Member(doc, "decoder"));
+ impl->byte_level_only = dynamic_cast<const ByteLevel*>(impl->pre.get()) != nullptr;
+
+ // Added tokens.
+ std::vector<AddedToken> added;
+ if (const Value* at = Member(doc, "added_tokens")) {
+ if (at->kind() != Value::Kind::Array) Refuse("added_tokens must be an array");
+ std::map<int32_t, std::string> seen;
+ for (const Value& r : at->items()) {
+ if (r.kind() != Value::Kind::Object) Refuse("an added_tokens entry must be an object");
+ std::string content = Str(r, "content", "");
+ if (content.empty()) Refuse("an added_tokens entry must declare content");
+ const Value* id = Member(r, "id");
+ if (id == nullptr || id->kind() != Value::Kind::Int || id->as_int() < 0 || id->as_int() > INT32_MAX) {
+ Refuse("added token \"" + content + "\" has no non-negative id");
+ }
+ int32_t i = int32_t(id->as_int());
+ auto clash = seen.find(i);
+ if (clash != seen.end() && clash->second != content) {
+ Refuse("added token id " + std::to_string(i) + " is claimed by both \"" + clash->second + "\" and \"" + content + "\"");
+ }
+ seen[i] = content;
+ added.push_back(AddedToken{content, i, Bool(r, "special", false), Bool(r, "lstrip", false), Bool(r, "rstrip", false),
+ Bool(r, "single_word", false), Bool(r, "normalized", false)});
+ }
+ }
+ const Value* vocab_v = Member(model, "vocab");
+ if (vocab_v == nullptr || vocab_v->kind() != Value::Kind::Object) Refuse("model.vocab must be an object");
+ auto& vocab = impl->ids;
+ vocab.reserve(vocab_v->size() + added.size());
+ for (size_t i = 0; i < vocab_v->size(); i++) {
+ const Value& v = vocab_v->value(i);
+ if (v.kind() != Value::Kind::Int || v.as_int() > INT32_MAX) Refuse("the vocabulary holds an id that is not an integer");
+ if (v.as_int() < 0) Refuse("the vocabulary holds a negative id " + std::to_string(v.as_int()));
+ vocab[vocab_v->key(i)] = int32_t(v.as_int());
+ }
+ for (const auto& tok : added) {
+ if (tok.special) {
+ impl->is_special.insert(tok.id);
+ impl->specials[tok.content] = tok.id;
+ }
+ vocab[tok.content] = tok.id; // the added-token id wins over the vocabulary's, as in the reference
+ }
+ int32_t highest = 0;
+ for (const auto& kv : vocab) highest = std::max(highest, kv.second);
+ // Ids may have gaps, but a dense table sized by a hostile id would be gigabytes.
+ if (int64_t(highest) >= int64_t(vocab.size()) * 4 + (1 << 16)) {
+ Refuse("the highest token id " + std::to_string(highest) + " is far beyond the " + std::to_string(vocab.size()) +
+ " tokens; ids this sparse are refused rather than allocated");
+ }
+ impl->tokens.assign(size_t(highest) + 1, std::string());
+ for (const auto& kv : vocab) {
+ std::string& owner = impl->tokens[size_t(kv.second)];
+ if (!owner.empty() && owner != kv.first) {
+ Refuse("the vocabulary gives id " + std::to_string(kv.second) + " to both \"" + owner + "\" and \"" + kv.first + "\"");
+ }
+ owner = kv.first;
+ }
+
+ std::vector<std::pair<std::string, std::string>> merges;
+ ReadMerges(model, merges);
+ std::string unk;
+ const Value* unk_v = Member(model, "unk_token");
+ if (unk_v != nullptr && unk_v->kind() != Value::Kind::String) Refuse("model.unk_token must be a string or null");
+ if (unk_v != nullptr) unk = unk_v->as_string();
+ impl->bpe = std::make_unique<Bpe>(vocab, merges, unk_v != nullptr ? &unk : nullptr, Bool(model, "fuse_unk", false),
+ Bool(model, "ignore_merges", false), Bool(model, "byte_fallback", false));
+
+ // Refuse a tokenizer that could drop text without trace.
+ if (!impl->bpe->has_unk() && !impl->bpe->has_byte_fallback()) {
+ if (!impl->byte_level_only) {
+ Unsupported("this checkpoint declares no unk_token and no byte_fallback, and its pre-tokenizer is not byte-level, "
+ "so a character outside the vocabulary would be dropped without trace. Refusing to load");
+ }
+ int missing = 0, reachable = 0;
+ std::string first;
+ for (int b = 0; b < 256; b++) {
+ if (b == 0xC0 || b == 0xC1 || b >= 0xF5) continue; // bytes UTF-8 never contains
+ reachable++;
+ std::string symbol;
+ ud::AppendRune(symbol, TheAlphabet().to_rune[b]);
+ if (vocab.find(symbol) == vocab.end()) {
+ if (missing == 0) first = symbol;
+ missing++;
+ }
+ }
+ if (missing > 0) {
+ Refuse("this checkpoint declares no unk_token and no byte_fallback, and " + std::to_string(missing) + " of the " +
+ std::to_string(reachable) + " byte-level alphabet symbols that text can reach are absent from its vocabulary "
+ "(first: \"" + first + "\"), so some inputs would lose text silently. Refusing to load");
+ }
+ }
+
+ std::vector<AddedToken> on_raw, on_normalized;
+ for (const auto& tok : added) (tok.normalized ? on_normalized : on_raw).push_back(tok);
+ impl->raw_added = AddedVocab(on_raw);
+ impl->normal_added = AddedVocab(on_normalized);
+
+ // Roles named by tokenizer_config.json.
+ if (config != nullptr) {
+ static const char* const kRoles[] = {"cls_token", "sep_token", "mask_token", "pad_token", "unk_token"};
+ for (const char* key : kRoles) {
+ const Value* v = Member(*config, key);
+ if (v == nullptr) continue;
+ std::string content;
+ if (v->kind() == Value::Kind::String) {
+ content = v->as_string();
+ } else if (v->kind() == Value::Kind::Object && v->Find("content") != nullptr &&
+ v->Find("content")->kind() == Value::Kind::String) {
+ content = v->Find("content")->as_string();
+ } else {
+ Refuse(std::string(key) + " in tokenizer_config.json must be a string or an object with a content field");
+ }
+ auto it = vocab.find(content);
+ if (it == vocab.end()) {
+ Refuse("tokenizer_config.json names " + content + " as " + key +
+ ", but neither the vocabulary nor the added-token table has it");
+ }
+ impl->roles[key] = it->second;
+ if (std::string(key) == "mask_token") impl->mask_token = content;
+ }
+ }
+ return impl;
+}
+
+} // namespace
+
+Result<Tokenizer> Tokenizer::Parse(std::string_view tokenizer_json, std::optional<std::string_view> config_json) {
+ try {
+ Value config;
+ const Value* cp = nullptr;
+ if (config_json.has_value()) {
+ auto c = pyvalue::Parse(*config_json);
+ if (!c.ok() || c.value().kind() != Value::Kind::Object) {
+ return Result<Tokenizer>::Fail("tokenizer_config.json must be a JSON object" + (c.ok() ? std::string() : ": " + c.error()));
+ }
+ config = std::move(c).value();
+ cp = &config;
+ }
+ return Result<Tokenizer>::Ok(Tokenizer(Build(tokenizer_json, cp)));
+ } catch (const Fail& f) {
+ return Result<Tokenizer>::Fail(f.msg);
+ } catch (const std::bad_alloc&) {
+ return Result<Tokenizer>::Fail("out of memory loading the tokenizer");
+ }
+}
+
+Result<Tokenizer> Tokenizer::LoadDir(const std::string& dir) {
+ auto read = [](const std::string& path, std::string* out) {
+ std::ifstream in(path, std::ios::binary);
+ if (!in) return false;
+ std::stringstream ss;
+ ss << in.rdbuf();
+ *out = ss.str();
+ return true;
+ };
+ std::string where = dir, json, config;
+ if (std::ifstream(dir + "/tokenizer/tokenizer.json").good()) where = dir + "/tokenizer";
+ if (!read(where + "/tokenizer.json", &json)) return Result<Tokenizer>::Fail("cannot read " + where + "/tokenizer.json");
+ bool has_config = read(where + "/tokenizer_config.json", &config);
+ return Parse(json, has_config ? std::optional<std::string_view>(config) : std::nullopt);
+}
+
+std::vector<int32_t> Tokenizer::Encode(std::string_view text_in, int64_t limit) const {
+ std::vector<int32_t> out;
+ if (limit == 0) return out;
+ const Impl& t = *impl_;
+ std::string text = ud::ToValid(text_in); // a Python str is always valid
+ out.reserve(size_t(limit > 0 && limit < int64_t(text.size() / 3 + 16) ? limit : int64_t(text.size() / 3 + 16)));
+ auto full = [&]() { return limit >= 0 && int64_t(out.size()) >= limit; };
+ auto plain = [&](const std::string& s) {
+ std::vector<std::string> pieces;
+ t.pre->Split(s, pieces);
+ for (const auto& piece : pieces) {
+ if (full()) return;
+ t.bpe->EncodePiece(piece, out);
+ }
+ };
+ for (const Span& sp : t.raw_added.Split(text)) {
+ if (full()) break;
+ if (sp.is_token) {
+ out.push_back(sp.id);
+ continue;
+ }
+ std::string normalized = t.normalizer(sp.text);
+ if (t.normal_added.empty) {
+ plain(normalized);
+ continue;
+ }
+ for (const Span& inner : t.normal_added.Split(normalized)) {
+ if (full()) break;
+ if (inner.is_token) out.push_back(inner.id);
+ else plain(inner.text);
+ }
+ }
+ if (limit >= 0 && int64_t(out.size()) > limit) out.resize(size_t(limit));
+ return out;
+}
+
+std::string Tokenizer::Decode(const std::vector<int32_t>& ids, bool skip_special_tokens) const {
+ const Impl& t = *impl_;
+ std::vector<std::string> pieces;
+ pieces.reserve(ids.size());
+ for (int32_t id : ids) {
+ if (skip_special_tokens && t.is_special.count(id) != 0) continue;
+ if (id < 0 || size_t(id) >= t.tokens.size() || t.tokens[size_t(id)].empty()) continue;
+ pieces.push_back(t.tokens[size_t(id)]);
+ }
+ std::string out;
+ if (t.decoder == nullptr) { // with no decoder section the reference joins tokens with a space
+ for (size_t i = 0; i < pieces.size(); i++) out += (i ? " " : "") + pieces[i];
+ return out;
+ }
+ t.decoder->Transform(pieces);
+ for (const auto& p : pieces) out += p;
+ return out;
+}
+
+size_t Tokenizer::VocabSize() const { return impl_->ids.size(); }
+int32_t Tokenizer::HighestId() const { return int32_t(impl_->tokens.size()) - 1; }
+std::optional<int32_t> Tokenizer::TokenToId(std::string_view token) const {
+ auto it = impl_->ids.find(std::string(token));
+ if (it == impl_->ids.end()) return std::nullopt;
+ return it->second;
+}
+std::optional<std::string> Tokenizer::IdToToken(int32_t id) const {
+ if (id < 0 || size_t(id) >= impl_->tokens.size() || impl_->tokens[size_t(id)].empty()) return std::nullopt;
+ return impl_->tokens[size_t(id)];
+}
+std::optional<int32_t> Tokenizer::SpecialId(std::string_view content) const {
+ auto it = impl_->specials.find(std::string(content));
+ if (it == impl_->specials.end()) return std::nullopt;
+ return it->second;
+}
+std::map<std::string, int32_t> Tokenizer::SpecialTokens() const { return impl_->specials; }
+static std::optional<int32_t> Role(const Impl& i, const char* key) {
+ auto it = i.roles.find(key);
+ if (it == i.roles.end()) return std::nullopt;
+ return it->second;
+}
+std::optional<int32_t> Tokenizer::Cls() const { return Role(*impl_, "cls_token"); }
+std::optional<int32_t> Tokenizer::Sep() const { return Role(*impl_, "sep_token"); }
+std::optional<int32_t> Tokenizer::Mask() const { return Role(*impl_, "mask_token"); }
+std::optional<int32_t> Tokenizer::Pad() const { return Role(*impl_, "pad_token"); }
+std::string Tokenizer::MaskToken() const { return impl_->mask_token; }
+
+std::string Tokenizer::NormalizeForTest(std::string_view text) const { return impl_->normalizer(ud::ToValid(text)); }
+std::vector<std::string> Tokenizer::PreTokenizeForTest(std::string_view normalized) const {
+ std::vector<std::string> out;
+ impl_->pre->Split(normalized, out);
+ return out;
+}
+
+std::string Normalize(std::string_view text, Form form) { return NormalizeImpl(text, form); }
+std::string Lowercase(std::string_view text) { return LowercaseImpl(text); }
+
+Result<std::string> RunNormalizerSpec(std::string_view spec_json, std::string_view input) {
+ try {
+ auto v = pyvalue::Parse(spec_json);
+ if (!v.ok()) return Result<std::string>::Fail(v.error());
+ return Result<std::string>::Ok(ParseNormalizer(&v.value())(std::string(input)));
+ } catch (const Fail& f) {
+ return Result<std::string>::Fail(f.msg);
+ }
+}
+
+Result<std::vector<std::string>> RunPreTokenizerSpec(std::string_view spec_json, std::string_view input) {
+ try {
+ auto v = pyvalue::Parse(spec_json);
+ if (!v.ok()) return Result<std::vector<std::string>>::Fail(v.error());
+ std::vector<std::string> out;
+ ParsePreTokenizer(&v.value())->Split(input, out);
+ return Result<std::vector<std::string>>::Ok(out);
+ } catch (const Fail& f) {
+ return Result<std::vector<std::string>>::Fail(f.msg);
+ }
+}
+
+} // namespace tokenizer
+} // namespace laya
diff --git a/laya-cpp/src/tokenizer_tables.inc b/laya-cpp/src/tokenizer_tables.inc
new file mode 100644
index 00000000..26233846
--- /dev/null
+++ b/laya-cpp/src/tokenizer_tables.inc
@@ -0,0 +1,2123 @@
+// Code generated by laya-go/scripts/gen_tables.py --lang cpp from tokenizers 0.23.2; DO NOT EDIT.
+//
+// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp
+
+inline constexpr const char* kTokenizerReference = "tokenizers 0.23.2";
+
+// letter is the pre-tokenizer pattern's \p{L}, probed from the reference.
+// 677 ranges covering 141028 code points.
+inline constexpr uint32_t k_letter[] = {
+ 65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736,
+ 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, 906, 908, 908,
+ 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1488, 1514,
+ 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788,
+ 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042,
+ 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190,
+ 2208, 2249, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448,
+ 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529,
+ 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614,
+ 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736,
+ 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832,
+ 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929,
+ 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980,
+ 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133,
+ 3160, 3162, 3165, 3165, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251,
+ 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386,
+ 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515,
+ 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722,
+ 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807,
+ 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189,
+ 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301,
+ 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749,
+ 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885,
+ 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866,
+ 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103,
+ 6108, 6108, 6176, 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509,
+ 6512, 6516, 6528, 6571, 6576, 6601, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6988,
+ 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7306, 7312, 7354,
+ 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965,
+ 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116,
+ 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180,
+ 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469,
+ 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521,
+ 8526, 8526, 8579, 8580, 11264, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559,
+ 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696,
+ 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823,
+ 12293, 12294, 12337, 12341, 12347, 12348, 12353, 12438, 12445, 12447, 12449, 12538, 12540,
+ 12543, 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903, 19968, 42124,
+ 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653, 42656,
+ 42725, 42775, 42783, 42786, 42888, 42891, 42957, 42960, 42961, 42963, 42963, 42965, 42972,
+ 42994, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250,
+ 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442,
+ 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588,
+ 43595, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709,
+ 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785,
+ 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44002,
+ 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275,
+ 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321,
+ 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136,
+ 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487,
+ 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599,
+ 65613, 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66368,
+ 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66560, 66717, 66736,
+ 66771, 66776, 66811, 66816, 66855, 66864, 66915, 66928, 66938, 66940, 66954, 66956, 66962,
+ 66964, 66965, 66967, 66977, 66979, 66993, 66995, 67001, 67003, 67004, 67008, 67059, 67072,
+ 67382, 67392, 67413, 67424, 67431, 67456, 67461, 67463, 67504, 67506, 67514, 67584, 67589,
+ 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712,
+ 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031,
+ 68096, 68096, 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288,
+ 68295, 68297, 68324, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680,
+ 68736, 68786, 68800, 68850, 68864, 68899, 68938, 68965, 68975, 68997, 69248, 69289, 69296,
+ 69297, 69314, 69316, 69376, 69404, 69415, 69415, 69424, 69445, 69488, 69505, 69552, 69572,
+ 69600, 69622, 69635, 69687, 69745, 69746, 69749, 69749, 69763, 69807, 69840, 69864, 69891,
+ 69926, 69956, 69956, 69959, 69959, 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084,
+ 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278, 70280,
+ 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416,
+ 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, 70493,
+ 70497, 70528, 70537, 70539, 70539, 70542, 70542, 70544, 70581, 70583, 70583, 70609, 70609,
+ 70611, 70611, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853, 70855,
+ 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352,
+ 71424, 71450, 71488, 71494, 71680, 71723, 71840, 71903, 71935, 71942, 71945, 71945, 71948,
+ 71955, 71957, 71958, 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144,
+ 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284,
+ 72329, 72349, 72349, 72368, 72440, 72640, 72672, 72704, 72712, 72714, 72750, 72768, 72768,
+ 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, 73063,
+ 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488, 73490, 73523,
+ 73648, 73648, 73728, 74649, 74880, 75075, 77712, 77808, 77824, 78895, 78913, 78918, 78944,
+ 82938, 82944, 83526, 90368, 90397, 92160, 92728, 92736, 92766, 92784, 92862, 92880, 92909,
+ 92928, 92975, 92992, 92995, 93027, 93047, 93053, 93071, 93504, 93548, 93760, 93823, 93952,
+ 94026, 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101589,
+ 101631, 101640, 110576, 110579, 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898,
+ 110928, 110930, 110933, 110933, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788,
+ 113792, 113800, 113808, 113817, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970,
+ 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069,
+ 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132,
+ 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570,
+ 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744,
+ 120746, 120770, 120772, 120779, 122624, 122654, 122661, 122666, 122928, 122989, 123136, 123180,
+ 123191, 123197, 123214, 123214, 123536, 123565, 123584, 123627, 124112, 124139, 124368, 124397,
+ 124400, 124400, 124896, 124902, 124904, 124907, 124909, 124910, 124912, 124926, 124928, 125124,
+ 125184, 125251, 125259, 125259, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500,
+ 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530,
+ 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548,
+ 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562,
+ 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590,
+ 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173791,
+ 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 191472, 192093, 194560, 195101,
+ 196608, 201546, 201552, 205743,
+};
+
+// number is the pre-tokenizer pattern's \p{N}, probed from the reference.
+// 144 ranges covering 1911 code points.
+inline constexpr uint32_t k_number[] = {
+ 48, 57, 178, 179, 185, 185, 188, 190, 1632, 1641, 1776, 1785, 1984, 1993, 2406, 2415, 2534,
+ 2543, 2548, 2553, 2662, 2671, 2790, 2799, 2918, 2927, 2930, 2935, 3046, 3058, 3174, 3183, 3192,
+ 3198, 3302, 3311, 3416, 3422, 3430, 3448, 3558, 3567, 3664, 3673, 3792, 3801, 3872, 3891, 4160,
+ 4169, 4240, 4249, 4969, 4988, 5870, 5872, 6112, 6121, 6128, 6137, 6160, 6169, 6470, 6479, 6608,
+ 6618, 6784, 6793, 6800, 6809, 6992, 7001, 7088, 7097, 7232, 7241, 7248, 7257, 8304, 8304, 8308,
+ 8313, 8320, 8329, 8528, 8578, 8581, 8585, 9312, 9371, 9450, 9471, 10102, 10131, 11517, 11517,
+ 12295, 12295, 12321, 12329, 12344, 12346, 12690, 12693, 12832, 12841, 12872, 12879, 12881,
+ 12895, 12928, 12937, 12977, 12991, 42528, 42537, 42726, 42735, 43056, 43061, 43216, 43225,
+ 43264, 43273, 43472, 43481, 43504, 43513, 43600, 43609, 44016, 44025, 65296, 65305, 65799,
+ 65843, 65856, 65912, 65930, 65931, 66273, 66299, 66336, 66339, 66369, 66369, 66378, 66378,
+ 66513, 66517, 66720, 66729, 67672, 67679, 67705, 67711, 67751, 67759, 67835, 67839, 67862,
+ 67867, 68028, 68029, 68032, 68047, 68050, 68095, 68160, 68168, 68221, 68222, 68253, 68255,
+ 68331, 68335, 68440, 68447, 68472, 68479, 68521, 68527, 68858, 68863, 68912, 68921, 68928,
+ 68937, 69216, 69246, 69405, 69414, 69457, 69460, 69573, 69579, 69714, 69743, 69872, 69881,
+ 69942, 69951, 70096, 70105, 70113, 70132, 70384, 70393, 70736, 70745, 70864, 70873, 71248,
+ 71257, 71360, 71369, 71376, 71395, 71472, 71483, 71904, 71922, 72016, 72025, 72688, 72697,
+ 72784, 72812, 73040, 73049, 73120, 73129, 73552, 73561, 73664, 73684, 74752, 74862, 90416,
+ 90425, 92768, 92777, 92864, 92873, 93008, 93017, 93019, 93025, 93552, 93561, 93824, 93846,
+ 118000, 118009, 119488, 119507, 119520, 119539, 119648, 119672, 120782, 120831, 123200, 123209,
+ 123632, 123641, 124144, 124153, 124401, 124410, 125127, 125135, 125264, 125273, 126065, 126123,
+ 126125, 126127, 126129, 126132, 126209, 126253, 126255, 126269, 127232, 127244, 130032, 130041,
+};
+
+// wordChar is what the reference counts as a word character beside a
+// single_word added token.
+// 796 ranges covering 144667 code points.
+inline constexpr uint32_t k_word_char[] = {
+ 48, 57, 65, 90, 95, 95, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705,
+ 710, 721, 736, 740, 748, 748, 750, 750, 768, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904,
+ 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1155, 1327, 1329, 1366, 1369, 1369, 1376, 1416,
+ 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1488, 1514, 1519, 1522, 1552, 1562,
+ 1568, 1641, 1646, 1747, 1749, 1756, 1759, 1768, 1770, 1788, 1791, 1791, 1808, 1866, 1869, 1969,
+ 1984, 2037, 2042, 2042, 2045, 2045, 2048, 2093, 2112, 2139, 2144, 2154, 2160, 2183, 2185, 2190,
+ 2199, 2273, 2275, 2403, 2406, 2415, 2417, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480,
+ 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531,
+ 2534, 2545, 2556, 2556, 2558, 2558, 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608,
+ 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641,
+ 2649, 2652, 2654, 2654, 2662, 2677, 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736,
+ 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2799,
+ 2809, 2815, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873,
+ 2876, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2908, 2909, 2911, 2915, 2918, 2927, 2929, 2929,
+ 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980,
+ 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3055,
+ 3072, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3132, 3140, 3142, 3144, 3146, 3149, 3157, 3158,
+ 3160, 3162, 3165, 3165, 3168, 3171, 3174, 3183, 3200, 3203, 3205, 3212, 3214, 3216, 3218, 3240,
+ 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3293, 3294, 3296, 3299,
+ 3302, 3311, 3313, 3315, 3328, 3340, 3342, 3344, 3346, 3396, 3398, 3400, 3402, 3406, 3412, 3415,
+ 3423, 3427, 3430, 3439, 3450, 3455, 3457, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517,
+ 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3558, 3567, 3570, 3571, 3585, 3642,
+ 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3773,
+ 3776, 3780, 3782, 3782, 3784, 3790, 3792, 3801, 3804, 3807, 3840, 3840, 3864, 3865, 3872, 3881,
+ 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3948, 3953, 3972, 3974, 3991, 3993, 4028,
+ 4038, 4038, 4096, 4169, 4176, 4253, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680,
+ 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789,
+ 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4959,
+ 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880,
+ 5888, 5909, 5919, 5940, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6099, 6103, 6103,
+ 6108, 6109, 6112, 6121, 6155, 6157, 6159, 6169, 6176, 6264, 6272, 6314, 6320, 6389, 6400, 6430,
+ 6432, 6443, 6448, 6459, 6470, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6617, 6656, 6683,
+ 6688, 6750, 6752, 6780, 6783, 6793, 6800, 6809, 6823, 6823, 6832, 6862, 6912, 6988, 6992, 7001,
+ 7019, 7027, 7040, 7155, 7168, 7223, 7232, 7241, 7245, 7293, 7296, 7306, 7312, 7354, 7357, 7359,
+ 7376, 7378, 7380, 7418, 7424, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025,
+ 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140,
+ 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8204, 8205, 8255, 8256, 8276, 8276,
+ 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8432, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469,
+ 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521,
+ 8526, 8526, 8544, 8584, 9398, 9449, 11264, 11492, 11499, 11507, 11520, 11557, 11559, 11559,
+ 11565, 11565, 11568, 11623, 11631, 11631, 11647, 11670, 11680, 11686, 11688, 11694, 11696,
+ 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11744, 11775,
+ 11823, 11823, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12348, 12353, 12438, 12441,
+ 12442, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12735,
+ 12784, 12799, 13312, 19903, 19968, 42124, 42192, 42237, 42240, 42508, 42512, 42539, 42560,
+ 42610, 42612, 42621, 42623, 42737, 42775, 42783, 42786, 42888, 42891, 42957, 42960, 42961,
+ 42963, 42963, 42965, 42972, 42994, 43047, 43052, 43052, 43072, 43123, 43136, 43205, 43216,
+ 43225, 43232, 43255, 43259, 43259, 43261, 43309, 43312, 43347, 43360, 43388, 43392, 43456,
+ 43471, 43481, 43488, 43518, 43520, 43574, 43584, 43597, 43600, 43609, 43616, 43638, 43642,
+ 43714, 43739, 43741, 43744, 43759, 43762, 43766, 43777, 43782, 43785, 43790, 43793, 43798,
+ 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44010, 44012, 44013, 44016,
+ 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262,
+ 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323,
+ 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65024, 65039,
+ 65056, 65071, 65075, 65076, 65101, 65103, 65136, 65140, 65142, 65276, 65296, 65305, 65313,
+ 65338, 65343, 65343, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495,
+ 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616,
+ 65629, 65664, 65786, 65856, 65908, 66045, 66045, 66176, 66204, 66208, 66256, 66272, 66272,
+ 66304, 66335, 66349, 66378, 66384, 66426, 66432, 66461, 66464, 66499, 66504, 66511, 66513,
+ 66517, 66560, 66717, 66720, 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915,
+ 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, 66995,
+ 67001, 67003, 67004, 67008, 67059, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461,
+ 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644,
+ 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861,
+ 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68099, 68101, 68102, 68108, 68115, 68117,
+ 68119, 68121, 68149, 68152, 68154, 68159, 68159, 68192, 68220, 68224, 68252, 68288, 68295,
+ 68297, 68326, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736,
+ 68786, 68800, 68850, 68864, 68903, 68912, 68921, 68928, 68965, 68969, 68973, 68975, 68997,
+ 69248, 69289, 69291, 69292, 69296, 69297, 69314, 69316, 69372, 69404, 69415, 69415, 69424,
+ 69456, 69488, 69509, 69552, 69572, 69600, 69622, 69632, 69702, 69734, 69749, 69759, 69818,
+ 69826, 69826, 69840, 69864, 69872, 69881, 69888, 69940, 69942, 69951, 69956, 69959, 69968,
+ 70003, 70006, 70006, 70016, 70084, 70089, 70092, 70094, 70106, 70108, 70108, 70144, 70161,
+ 70163, 70199, 70206, 70209, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303,
+ 70312, 70320, 70378, 70384, 70393, 70400, 70403, 70405, 70412, 70415, 70416, 70419, 70440,
+ 70442, 70448, 70450, 70451, 70453, 70457, 70459, 70468, 70471, 70472, 70475, 70477, 70480,
+ 70480, 70487, 70487, 70493, 70499, 70502, 70508, 70512, 70516, 70528, 70537, 70539, 70539,
+ 70542, 70542, 70544, 70581, 70583, 70592, 70594, 70594, 70597, 70597, 70599, 70602, 70604,
+ 70611, 70625, 70626, 70656, 70730, 70736, 70745, 70750, 70753, 70784, 70853, 70855, 70855,
+ 70864, 70873, 71040, 71093, 71096, 71104, 71128, 71133, 71168, 71232, 71236, 71236, 71248,
+ 71257, 71296, 71352, 71360, 71369, 71376, 71395, 71424, 71450, 71453, 71467, 71472, 71481,
+ 71488, 71494, 71680, 71738, 71840, 71913, 71935, 71942, 71945, 71945, 71948, 71955, 71957,
+ 71958, 71960, 71989, 71991, 71992, 71995, 72003, 72016, 72025, 72096, 72103, 72106, 72151,
+ 72154, 72161, 72163, 72164, 72192, 72254, 72263, 72263, 72272, 72345, 72349, 72349, 72368,
+ 72440, 72640, 72672, 72688, 72697, 72704, 72712, 72714, 72758, 72760, 72768, 72784, 72793,
+ 72818, 72847, 72850, 72871, 72873, 72886, 72960, 72966, 72968, 72969, 72971, 73014, 73018,
+ 73018, 73020, 73021, 73023, 73031, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73102,
+ 73104, 73105, 73107, 73112, 73120, 73129, 73440, 73462, 73472, 73488, 73490, 73530, 73534,
+ 73538, 73552, 73562, 73648, 73648, 73728, 74649, 74752, 74862, 74880, 75075, 77712, 77808,
+ 77824, 78895, 78912, 78933, 78944, 82938, 82944, 83526, 90368, 90425, 92160, 92728, 92736,
+ 92766, 92768, 92777, 92784, 92862, 92864, 92873, 92880, 92909, 92912, 92916, 92928, 92982,
+ 92992, 92995, 93008, 93017, 93027, 93047, 93053, 93071, 93504, 93548, 93552, 93561, 93760,
+ 93823, 93952, 94026, 94031, 94087, 94095, 94111, 94176, 94177, 94179, 94180, 94192, 94193,
+ 94208, 100343, 100352, 101589, 101631, 101640, 110576, 110579, 110581, 110587, 110589, 110590,
+ 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951, 110960, 111355,
+ 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 113821, 113822, 118000, 118009,
+ 118528, 118573, 118576, 118598, 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179,
+ 119210, 119213, 119362, 119364, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970,
+ 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069,
+ 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132,
+ 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570,
+ 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744,
+ 120746, 120770, 120772, 120779, 120782, 120831, 121344, 121398, 121403, 121452, 121461, 121461,
+ 121476, 121476, 121499, 121503, 121505, 121519, 122624, 122654, 122661, 122666, 122880, 122886,
+ 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 122928, 122989, 123023, 123023,
+ 123136, 123180, 123184, 123197, 123200, 123209, 123214, 123214, 123536, 123566, 123584, 123641,
+ 124112, 124153, 124368, 124410, 124896, 124902, 124904, 124907, 124909, 124910, 124912, 124926,
+ 124928, 125124, 125136, 125142, 125184, 125259, 125264, 125273, 126464, 126467, 126469, 126495,
+ 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521,
+ 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543,
+ 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557,
+ 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583,
+ 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633,
+ 126635, 126651, 127280, 127305, 127312, 127337, 127344, 127369, 130032, 130041, 131072, 173791,
+ 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 191472, 192093, 194560, 195101,
+ 196608, 201546, 201552, 205743, 917760, 917999,
+};
+
+// lowercase, one code point to one
+inline constexpr uint32_t k_lower_from[] = {
+ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
+ 89, 90, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208,
+ 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, 221, 222, 256, 258, 260, 262, 264, 266,
+ 268, 270, 272, 274, 276, 278, 280, 282, 284, 286, 288, 290, 292, 294, 296, 298, 300, 302, 306,
+ 308, 310, 313, 315, 317, 319, 321, 323, 325, 327, 330, 332, 334, 336, 338, 340, 342, 344, 346,
+ 348, 350, 352, 354, 356, 358, 360, 362, 364, 366, 368, 370, 372, 374, 376, 377, 379, 381, 385,
+ 386, 388, 390, 391, 393, 394, 395, 398, 399, 400, 401, 403, 404, 406, 407, 408, 412, 413, 415,
+ 416, 418, 420, 422, 423, 425, 428, 430, 431, 433, 434, 435, 437, 439, 440, 444, 452, 453, 455,
+ 456, 458, 459, 461, 463, 465, 467, 469, 471, 473, 475, 478, 480, 482, 484, 486, 488, 490, 492,
+ 494, 497, 498, 500, 502, 503, 504, 506, 508, 510, 512, 514, 516, 518, 520, 522, 524, 526, 528,
+ 530, 532, 534, 536, 538, 540, 542, 544, 546, 548, 550, 552, 554, 556, 558, 560, 562, 570, 571,
+ 573, 574, 577, 579, 580, 581, 582, 584, 586, 588, 590, 880, 882, 886, 895, 902, 904, 905, 906,
+ 908, 910, 911, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928,
+ 929, 931, 932, 933, 934, 935, 936, 937, 938, 939, 975, 984, 986, 988, 990, 992, 994, 996, 998,
+ 1000, 1002, 1004, 1006, 1012, 1015, 1017, 1018, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028,
+ 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044,
+ 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060,
+ 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1120, 1122, 1124, 1126, 1128,
+ 1130, 1132, 1134, 1136, 1138, 1140, 1142, 1144, 1146, 1148, 1150, 1152, 1162, 1164, 1166, 1168,
+ 1170, 1172, 1174, 1176, 1178, 1180, 1182, 1184, 1186, 1188, 1190, 1192, 1194, 1196, 1198, 1200,
+ 1202, 1204, 1206, 1208, 1210, 1212, 1214, 1216, 1217, 1219, 1221, 1223, 1225, 1227, 1229, 1232,
+ 1234, 1236, 1238, 1240, 1242, 1244, 1246, 1248, 1250, 1252, 1254, 1256, 1258, 1260, 1262, 1264,
+ 1266, 1268, 1270, 1272, 1274, 1276, 1278, 1280, 1282, 1284, 1286, 1288, 1290, 1292, 1294, 1296,
+ 1298, 1300, 1302, 1304, 1306, 1308, 1310, 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, 1329,
+ 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1345,
+ 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361,
+ 1362, 1363, 1364, 1365, 1366, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266,
+ 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282,
+ 4283, 4284, 4285, 4286, 4287, 4288, 4289, 4290, 4291, 4292, 4293, 4295, 4301, 5024, 5025, 5026,
+ 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041, 5042,
+ 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5054, 5055, 5056, 5057, 5058,
+ 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073, 5074,
+ 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5089, 5090,
+ 5091, 5092, 5093, 5094, 5095, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106,
+ 5107, 5108, 5109, 7305, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, 7322, 7323,
+ 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339,
+ 7340, 7341, 7342, 7343, 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7357,
+ 7358, 7359, 7680, 7682, 7684, 7686, 7688, 7690, 7692, 7694, 7696, 7698, 7700, 7702, 7704, 7706,
+ 7708, 7710, 7712, 7714, 7716, 7718, 7720, 7722, 7724, 7726, 7728, 7730, 7732, 7734, 7736, 7738,
+ 7740, 7742, 7744, 7746, 7748, 7750, 7752, 7754, 7756, 7758, 7760, 7762, 7764, 7766, 7768, 7770,
+ 7772, 7774, 7776, 7778, 7780, 7782, 7784, 7786, 7788, 7790, 7792, 7794, 7796, 7798, 7800, 7802,
+ 7804, 7806, 7808, 7810, 7812, 7814, 7816, 7818, 7820, 7822, 7824, 7826, 7828, 7838, 7840, 7842,
+ 7844, 7846, 7848, 7850, 7852, 7854, 7856, 7858, 7860, 7862, 7864, 7866, 7868, 7870, 7872, 7874,
+ 7876, 7878, 7880, 7882, 7884, 7886, 7888, 7890, 7892, 7894, 7896, 7898, 7900, 7902, 7904, 7906,
+ 7908, 7910, 7912, 7914, 7916, 7918, 7920, 7922, 7924, 7926, 7928, 7930, 7932, 7934, 7944, 7945,
+ 7946, 7947, 7948, 7949, 7950, 7951, 7960, 7961, 7962, 7963, 7964, 7965, 7976, 7977, 7978, 7979,
+ 7980, 7981, 7982, 7983, 7992, 7993, 7994, 7995, 7996, 7997, 7998, 7999, 8008, 8009, 8010, 8011,
+ 8012, 8013, 8025, 8027, 8029, 8031, 8040, 8041, 8042, 8043, 8044, 8045, 8046, 8047, 8072, 8073,
+ 8074, 8075, 8076, 8077, 8078, 8079, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8104, 8105,
+ 8106, 8107, 8108, 8109, 8110, 8111, 8120, 8121, 8122, 8123, 8124, 8136, 8137, 8138, 8139, 8140,
+ 8152, 8153, 8154, 8155, 8168, 8169, 8170, 8171, 8172, 8184, 8185, 8186, 8187, 8188, 8486, 8490,
+ 8491, 8498, 8544, 8545, 8546, 8547, 8548, 8549, 8550, 8551, 8552, 8553, 8554, 8555, 8556, 8557,
+ 8558, 8559, 8579, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410,
+ 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, 11264, 11265,
+ 11266, 11267, 11268, 11269, 11270, 11271, 11272, 11273, 11274, 11275, 11276, 11277, 11278,
+ 11279, 11280, 11281, 11282, 11283, 11284, 11285, 11286, 11287, 11288, 11289, 11290, 11291,
+ 11292, 11293, 11294, 11295, 11296, 11297, 11298, 11299, 11300, 11301, 11302, 11303, 11304,
+ 11305, 11306, 11307, 11308, 11309, 11310, 11311, 11360, 11362, 11363, 11364, 11367, 11369,
+ 11371, 11373, 11374, 11375, 11376, 11378, 11381, 11390, 11391, 11392, 11394, 11396, 11398,
+ 11400, 11402, 11404, 11406, 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424,
+ 11426, 11428, 11430, 11432, 11434, 11436, 11438, 11440, 11442, 11444, 11446, 11448, 11450,
+ 11452, 11454, 11456, 11458, 11460, 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476,
+ 11478, 11480, 11482, 11484, 11486, 11488, 11490, 11499, 11501, 11506, 42560, 42562, 42564,
+ 42566, 42568, 42570, 42572, 42574, 42576, 42578, 42580, 42582, 42584, 42586, 42588, 42590,
+ 42592, 42594, 42596, 42598, 42600, 42602, 42604, 42624, 42626, 42628, 42630, 42632, 42634,
+ 42636, 42638, 42640, 42642, 42644, 42646, 42648, 42650, 42786, 42788, 42790, 42792, 42794,
+ 42796, 42798, 42802, 42804, 42806, 42808, 42810, 42812, 42814, 42816, 42818, 42820, 42822,
+ 42824, 42826, 42828, 42830, 42832, 42834, 42836, 42838, 42840, 42842, 42844, 42846, 42848,
+ 42850, 42852, 42854, 42856, 42858, 42860, 42862, 42873, 42875, 42877, 42878, 42880, 42882,
+ 42884, 42886, 42891, 42893, 42896, 42898, 42902, 42904, 42906, 42908, 42910, 42912, 42914,
+ 42916, 42918, 42920, 42922, 42923, 42924, 42925, 42926, 42928, 42929, 42930, 42931, 42932,
+ 42934, 42936, 42938, 42940, 42942, 42944, 42946, 42948, 42949, 42950, 42951, 42953, 42955,
+ 42956, 42958, 42960, 42962, 42964, 42966, 42968, 42970, 42972, 42997, 65313, 65314, 65315,
+ 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323, 65324, 65325, 65326, 65327, 65328,
+ 65329, 65330, 65331, 65332, 65333, 65334, 65335, 65336, 65337, 65338, 66560, 66561, 66562,
+ 66563, 66564, 66565, 66566, 66567, 66568, 66569, 66570, 66571, 66572, 66573, 66574, 66575,
+ 66576, 66577, 66578, 66579, 66580, 66581, 66582, 66583, 66584, 66585, 66586, 66587, 66588,
+ 66589, 66590, 66591, 66592, 66593, 66594, 66595, 66596, 66597, 66598, 66599, 66736, 66737,
+ 66738, 66739, 66740, 66741, 66742, 66743, 66744, 66745, 66746, 66747, 66748, 66749, 66750,
+ 66751, 66752, 66753, 66754, 66755, 66756, 66757, 66758, 66759, 66760, 66761, 66762, 66763,
+ 66764, 66765, 66766, 66767, 66768, 66769, 66770, 66771, 66928, 66929, 66930, 66931, 66932,
+ 66933, 66934, 66935, 66936, 66937, 66938, 66940, 66941, 66942, 66943, 66944, 66945, 66946,
+ 66947, 66948, 66949, 66950, 66951, 66952, 66953, 66954, 66956, 66957, 66958, 66959, 66960,
+ 66961, 66962, 66964, 66965, 68736, 68737, 68738, 68739, 68740, 68741, 68742, 68743, 68744,
+ 68745, 68746, 68747, 68748, 68749, 68750, 68751, 68752, 68753, 68754, 68755, 68756, 68757,
+ 68758, 68759, 68760, 68761, 68762, 68763, 68764, 68765, 68766, 68767, 68768, 68769, 68770,
+ 68771, 68772, 68773, 68774, 68775, 68776, 68777, 68778, 68779, 68780, 68781, 68782, 68783,
+ 68784, 68785, 68786, 68944, 68945, 68946, 68947, 68948, 68949, 68950, 68951, 68952, 68953,
+ 68954, 68955, 68956, 68957, 68958, 68959, 68960, 68961, 68962, 68963, 68964, 68965, 71840,
+ 71841, 71842, 71843, 71844, 71845, 71846, 71847, 71848, 71849, 71850, 71851, 71852, 71853,
+ 71854, 71855, 71856, 71857, 71858, 71859, 71860, 71861, 71862, 71863, 71864, 71865, 71866,
+ 71867, 71868, 71869, 71870, 71871, 93760, 93761, 93762, 93763, 93764, 93765, 93766, 93767,
+ 93768, 93769, 93770, 93771, 93772, 93773, 93774, 93775, 93776, 93777, 93778, 93779, 93780,
+ 93781, 93782, 93783, 93784, 93785, 93786, 93787, 93788, 93789, 93790, 93791, 93856, 93857,
+ 93858, 93859, 93860, 93861, 93862, 93863, 93864, 93865, 93866, 93867, 93868, 93869, 93870,
+ 93871, 93872, 93873, 93874, 93875, 93876, 93877, 93878, 93879, 93880, 125184, 125185, 125186,
+ 125187, 125188, 125189, 125190, 125191, 125192, 125193, 125194, 125195, 125196, 125197, 125198,
+ 125199, 125200, 125201, 125202, 125203, 125204, 125205, 125206, 125207, 125208, 125209, 125210,
+ 125211, 125212, 125213, 125214, 125215, 125216, 125217,
+};
+
+// the lowercase of k_lower_from
+inline constexpr uint32_t k_lower_to[] = {
+ 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,
+ 116, 117, 118, 119, 120, 121, 122, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235,
+ 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 248, 249, 250, 251, 252, 253, 254, 257,
+ 259, 261, 263, 265, 267, 269, 271, 273, 275, 277, 279, 281, 283, 285, 287, 289, 291, 293, 295,
+ 297, 299, 301, 303, 307, 309, 311, 314, 316, 318, 320, 322, 324, 326, 328, 331, 333, 335, 337,
+ 339, 341, 343, 345, 347, 349, 351, 353, 355, 357, 359, 361, 363, 365, 367, 369, 371, 373, 375,
+ 255, 378, 380, 382, 595, 387, 389, 596, 392, 598, 599, 396, 477, 601, 603, 402, 608, 611, 617,
+ 616, 409, 623, 626, 629, 417, 419, 421, 640, 424, 643, 429, 648, 432, 650, 651, 436, 438, 658,
+ 441, 445, 454, 454, 457, 457, 460, 460, 462, 464, 466, 468, 470, 472, 474, 476, 479, 481, 483,
+ 485, 487, 489, 491, 493, 495, 499, 499, 501, 405, 447, 505, 507, 509, 511, 513, 515, 517, 519,
+ 521, 523, 525, 527, 529, 531, 533, 535, 537, 539, 541, 543, 414, 547, 549, 551, 553, 555, 557,
+ 559, 561, 563, 11365, 572, 410, 11366, 578, 384, 649, 652, 583, 585, 587, 589, 591, 881, 883,
+ 887, 1011, 940, 941, 942, 943, 972, 973, 974, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954,
+ 955, 956, 957, 958, 959, 960, 961, 963, 964, 965, 966, 967, 968, 969, 970, 971, 983, 985, 987,
+ 989, 991, 993, 995, 997, 999, 1001, 1003, 1005, 1007, 952, 1016, 1010, 1019, 891, 892, 893,
+ 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119,
+ 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087,
+ 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103,
+ 1121, 1123, 1125, 1127, 1129, 1131, 1133, 1135, 1137, 1139, 1141, 1143, 1145, 1147, 1149, 1151,
+ 1153, 1163, 1165, 1167, 1169, 1171, 1173, 1175, 1177, 1179, 1181, 1183, 1185, 1187, 1189, 1191,
+ 1193, 1195, 1197, 1199, 1201, 1203, 1205, 1207, 1209, 1211, 1213, 1215, 1231, 1218, 1220, 1222,
+ 1224, 1226, 1228, 1230, 1233, 1235, 1237, 1239, 1241, 1243, 1245, 1247, 1249, 1251, 1253, 1255,
+ 1257, 1259, 1261, 1263, 1265, 1267, 1269, 1271, 1273, 1275, 1277, 1279, 1281, 1283, 1285, 1287,
+ 1289, 1291, 1293, 1295, 1297, 1299, 1301, 1303, 1305, 1307, 1309, 1311, 1313, 1315, 1317, 1319,
+ 1321, 1323, 1325, 1327, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388,
+ 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404,
+ 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, 1414, 11520, 11521, 11522, 11523, 11524,
+ 11525, 11526, 11527, 11528, 11529, 11530, 11531, 11532, 11533, 11534, 11535, 11536, 11537,
+ 11538, 11539, 11540, 11541, 11542, 11543, 11544, 11545, 11546, 11547, 11548, 11549, 11550,
+ 11551, 11552, 11553, 11554, 11555, 11556, 11557, 11559, 11565, 43888, 43889, 43890, 43891,
+ 43892, 43893, 43894, 43895, 43896, 43897, 43898, 43899, 43900, 43901, 43902, 43903, 43904,
+ 43905, 43906, 43907, 43908, 43909, 43910, 43911, 43912, 43913, 43914, 43915, 43916, 43917,
+ 43918, 43919, 43920, 43921, 43922, 43923, 43924, 43925, 43926, 43927, 43928, 43929, 43930,
+ 43931, 43932, 43933, 43934, 43935, 43936, 43937, 43938, 43939, 43940, 43941, 43942, 43943,
+ 43944, 43945, 43946, 43947, 43948, 43949, 43950, 43951, 43952, 43953, 43954, 43955, 43956,
+ 43957, 43958, 43959, 43960, 43961, 43962, 43963, 43964, 43965, 43966, 43967, 5112, 5113, 5114,
+ 5115, 5116, 5117, 7306, 4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315,
+ 4316, 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325, 4326, 4327, 4328, 4329, 4330, 4331,
+ 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4349,
+ 4350, 4351, 7681, 7683, 7685, 7687, 7689, 7691, 7693, 7695, 7697, 7699, 7701, 7703, 7705, 7707,
+ 7709, 7711, 7713, 7715, 7717, 7719, 7721, 7723, 7725, 7727, 7729, 7731, 7733, 7735, 7737, 7739,
+ 7741, 7743, 7745, 7747, 7749, 7751, 7753, 7755, 7757, 7759, 7761, 7763, 7765, 7767, 7769, 7771,
+ 7773, 7775, 7777, 7779, 7781, 7783, 7785, 7787, 7789, 7791, 7793, 7795, 7797, 7799, 7801, 7803,
+ 7805, 7807, 7809, 7811, 7813, 7815, 7817, 7819, 7821, 7823, 7825, 7827, 7829, 223, 7841, 7843,
+ 7845, 7847, 7849, 7851, 7853, 7855, 7857, 7859, 7861, 7863, 7865, 7867, 7869, 7871, 7873, 7875,
+ 7877, 7879, 7881, 7883, 7885, 7887, 7889, 7891, 7893, 7895, 7897, 7899, 7901, 7903, 7905, 7907,
+ 7909, 7911, 7913, 7915, 7917, 7919, 7921, 7923, 7925, 7927, 7929, 7931, 7933, 7935, 7936, 7937,
+ 7938, 7939, 7940, 7941, 7942, 7943, 7952, 7953, 7954, 7955, 7956, 7957, 7968, 7969, 7970, 7971,
+ 7972, 7973, 7974, 7975, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991, 8000, 8001, 8002, 8003,
+ 8004, 8005, 8017, 8019, 8021, 8023, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, 8064, 8065,
+ 8066, 8067, 8068, 8069, 8070, 8071, 8080, 8081, 8082, 8083, 8084, 8085, 8086, 8087, 8096, 8097,
+ 8098, 8099, 8100, 8101, 8102, 8103, 8112, 8113, 8048, 8049, 8115, 8050, 8051, 8052, 8053, 8131,
+ 8144, 8145, 8054, 8055, 8160, 8161, 8058, 8059, 8165, 8056, 8057, 8060, 8061, 8179, 969, 107,
+ 229, 8526, 8560, 8561, 8562, 8563, 8564, 8565, 8566, 8567, 8568, 8569, 8570, 8571, 8572, 8573,
+ 8574, 8575, 8580, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436,
+ 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9448, 9449, 11312, 11313,
+ 11314, 11315, 11316, 11317, 11318, 11319, 11320, 11321, 11322, 11323, 11324, 11325, 11326,
+ 11327, 11328, 11329, 11330, 11331, 11332, 11333, 11334, 11335, 11336, 11337, 11338, 11339,
+ 11340, 11341, 11342, 11343, 11344, 11345, 11346, 11347, 11348, 11349, 11350, 11351, 11352,
+ 11353, 11354, 11355, 11356, 11357, 11358, 11359, 11361, 619, 7549, 637, 11368, 11370, 11372,
+ 593, 625, 592, 594, 11379, 11382, 575, 576, 11393, 11395, 11397, 11399, 11401, 11403, 11405,
+ 11407, 11409, 11411, 11413, 11415, 11417, 11419, 11421, 11423, 11425, 11427, 11429, 11431,
+ 11433, 11435, 11437, 11439, 11441, 11443, 11445, 11447, 11449, 11451, 11453, 11455, 11457,
+ 11459, 11461, 11463, 11465, 11467, 11469, 11471, 11473, 11475, 11477, 11479, 11481, 11483,
+ 11485, 11487, 11489, 11491, 11500, 11502, 11507, 42561, 42563, 42565, 42567, 42569, 42571,
+ 42573, 42575, 42577, 42579, 42581, 42583, 42585, 42587, 42589, 42591, 42593, 42595, 42597,
+ 42599, 42601, 42603, 42605, 42625, 42627, 42629, 42631, 42633, 42635, 42637, 42639, 42641,
+ 42643, 42645, 42647, 42649, 42651, 42787, 42789, 42791, 42793, 42795, 42797, 42799, 42803,
+ 42805, 42807, 42809, 42811, 42813, 42815, 42817, 42819, 42821, 42823, 42825, 42827, 42829,
+ 42831, 42833, 42835, 42837, 42839, 42841, 42843, 42845, 42847, 42849, 42851, 42853, 42855,
+ 42857, 42859, 42861, 42863, 42874, 42876, 7545, 42879, 42881, 42883, 42885, 42887, 42892, 613,
+ 42897, 42899, 42903, 42905, 42907, 42909, 42911, 42913, 42915, 42917, 42919, 42921, 614, 604,
+ 609, 620, 618, 670, 647, 669, 43859, 42933, 42935, 42937, 42939, 42941, 42943, 42945, 42947,
+ 42900, 642, 7566, 42952, 42954, 612, 42957, 42959, 42961, 42963, 42965, 42967, 42969, 42971,
+ 411, 42998, 65345, 65346, 65347, 65348, 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356,
+ 65357, 65358, 65359, 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, 65369,
+ 65370, 66600, 66601, 66602, 66603, 66604, 66605, 66606, 66607, 66608, 66609, 66610, 66611,
+ 66612, 66613, 66614, 66615, 66616, 66617, 66618, 66619, 66620, 66621, 66622, 66623, 66624,
+ 66625, 66626, 66627, 66628, 66629, 66630, 66631, 66632, 66633, 66634, 66635, 66636, 66637,
+ 66638, 66639, 66776, 66777, 66778, 66779, 66780, 66781, 66782, 66783, 66784, 66785, 66786,
+ 66787, 66788, 66789, 66790, 66791, 66792, 66793, 66794, 66795, 66796, 66797, 66798, 66799,
+ 66800, 66801, 66802, 66803, 66804, 66805, 66806, 66807, 66808, 66809, 66810, 66811, 66967,
+ 66968, 66969, 66970, 66971, 66972, 66973, 66974, 66975, 66976, 66977, 66979, 66980, 66981,
+ 66982, 66983, 66984, 66985, 66986, 66987, 66988, 66989, 66990, 66991, 66992, 66993, 66995,
+ 66996, 66997, 66998, 66999, 67000, 67001, 67003, 67004, 68800, 68801, 68802, 68803, 68804,
+ 68805, 68806, 68807, 68808, 68809, 68810, 68811, 68812, 68813, 68814, 68815, 68816, 68817,
+ 68818, 68819, 68820, 68821, 68822, 68823, 68824, 68825, 68826, 68827, 68828, 68829, 68830,
+ 68831, 68832, 68833, 68834, 68835, 68836, 68837, 68838, 68839, 68840, 68841, 68842, 68843,
+ 68844, 68845, 68846, 68847, 68848, 68849, 68850, 68976, 68977, 68978, 68979, 68980, 68981,
+ 68982, 68983, 68984, 68985, 68986, 68987, 68988, 68989, 68990, 68991, 68992, 68993, 68994,
+ 68995, 68996, 68997, 71872, 71873, 71874, 71875, 71876, 71877, 71878, 71879, 71880, 71881,
+ 71882, 71883, 71884, 71885, 71886, 71887, 71888, 71889, 71890, 71891, 71892, 71893, 71894,
+ 71895, 71896, 71897, 71898, 71899, 71900, 71901, 71902, 71903, 93792, 93793, 93794, 93795,
+ 93796, 93797, 93798, 93799, 93800, 93801, 93802, 93803, 93804, 93805, 93806, 93807, 93808,
+ 93809, 93810, 93811, 93812, 93813, 93814, 93815, 93816, 93817, 93818, 93819, 93820, 93821,
+ 93822, 93823, 93883, 93884, 93885, 93886, 93887, 93888, 93889, 93890, 93891, 93892, 93893,
+ 93894, 93895, 93896, 93897, 93898, 93899, 93900, 93901, 93902, 93903, 93904, 93905, 93906,
+ 93907, 125218, 125219, 125220, 125221, 125222, 125223, 125224, 125225, 125226, 125227, 125228,
+ 125229, 125230, 125231, 125232, 125233, 125234, 125235, 125236, 125237, 125238, 125239, 125240,
+ 125241, 125242, 125243, 125244, 125245, 125246, 125247, 125248, 125249, 125250, 125251,
+};
+
+struct TokLowerMulti { uint32_t from; uint32_t count; uint32_t to[3]; };
+inline constexpr TokLowerMulti k_lower_multi[] = {
+ {0x130, 2, {0x69, 0x307, 0x0}},
+};
+
+// canon* hold the full canonical decomposition (NFD) of every code
+// point that has one, except Hangul syllables, which are algorithmic.
+inline constexpr uint32_t k_canon_keys[] = {
+ 192, 193, 194, 195, 196, 197, 199, 200, 201, 202, 203, 204, 205, 206, 207, 209, 210, 211, 212,
+ 213, 214, 217, 218, 219, 220, 221, 224, 225, 226, 227, 228, 229, 231, 232, 233, 234, 235, 236,
+ 237, 238, 239, 241, 242, 243, 244, 245, 246, 249, 250, 251, 252, 253, 255, 256, 257, 258, 259,
+ 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 274, 275, 276, 277, 278, 279, 280,
+ 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 296, 297, 298, 299, 300, 301,
+ 302, 303, 304, 308, 309, 310, 311, 313, 314, 315, 316, 317, 318, 323, 324, 325, 326, 327, 328,
+ 332, 333, 334, 335, 336, 337, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352,
+ 353, 354, 355, 356, 357, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373,
+ 374, 375, 376, 377, 378, 379, 380, 381, 382, 416, 417, 431, 432, 461, 462, 463, 464, 465, 466,
+ 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 478, 479, 480, 481, 482, 483, 486, 487, 488,
+ 489, 490, 491, 492, 493, 494, 495, 496, 500, 501, 504, 505, 506, 507, 508, 509, 510, 511, 512,
+ 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531,
+ 532, 533, 534, 535, 536, 537, 538, 539, 542, 543, 550, 551, 552, 553, 554, 555, 556, 557, 558,
+ 559, 560, 561, 562, 563, 832, 833, 835, 836, 884, 894, 901, 902, 903, 904, 905, 906, 908, 910,
+ 911, 912, 938, 939, 940, 941, 942, 943, 944, 970, 971, 972, 973, 974, 979, 980, 1024, 1025,
+ 1027, 1031, 1036, 1037, 1038, 1049, 1081, 1104, 1105, 1107, 1111, 1116, 1117, 1118, 1142, 1143,
+ 1217, 1218, 1232, 1233, 1234, 1235, 1238, 1239, 1242, 1243, 1244, 1245, 1246, 1247, 1250, 1251,
+ 1252, 1253, 1254, 1255, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269,
+ 1272, 1273, 1570, 1571, 1572, 1573, 1574, 1728, 1730, 1747, 2345, 2353, 2356, 2392, 2393, 2394,
+ 2395, 2396, 2397, 2398, 2399, 2507, 2508, 2524, 2525, 2527, 2611, 2614, 2649, 2650, 2651, 2654,
+ 2888, 2891, 2892, 2908, 2909, 2964, 3018, 3019, 3020, 3144, 3264, 3271, 3272, 3274, 3275, 3402,
+ 3403, 3404, 3546, 3548, 3549, 3550, 3907, 3917, 3922, 3927, 3932, 3945, 3955, 3957, 3958, 3960,
+ 3969, 3987, 3997, 4002, 4007, 4012, 4025, 4134, 6918, 6920, 6922, 6924, 6926, 6930, 6971, 6973,
+ 6976, 6977, 6979, 7680, 7681, 7682, 7683, 7684, 7685, 7686, 7687, 7688, 7689, 7690, 7691, 7692,
+ 7693, 7694, 7695, 7696, 7697, 7698, 7699, 7700, 7701, 7702, 7703, 7704, 7705, 7706, 7707, 7708,
+ 7709, 7710, 7711, 7712, 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724,
+ 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, 7737, 7738, 7739, 7740,
+ 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756,
+ 7757, 7758, 7759, 7760, 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772,
+ 7773, 7774, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, 7785, 7786, 7787, 7788,
+ 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, 7797, 7798, 7799, 7800, 7801, 7802, 7803, 7804,
+ 7805, 7806, 7807, 7808, 7809, 7810, 7811, 7812, 7813, 7814, 7815, 7816, 7817, 7818, 7819, 7820,
+ 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, 7833, 7835, 7840, 7841,
+ 7842, 7843, 7844, 7845, 7846, 7847, 7848, 7849, 7850, 7851, 7852, 7853, 7854, 7855, 7856, 7857,
+ 7858, 7859, 7860, 7861, 7862, 7863, 7864, 7865, 7866, 7867, 7868, 7869, 7870, 7871, 7872, 7873,
+ 7874, 7875, 7876, 7877, 7878, 7879, 7880, 7881, 7882, 7883, 7884, 7885, 7886, 7887, 7888, 7889,
+ 7890, 7891, 7892, 7893, 7894, 7895, 7896, 7897, 7898, 7899, 7900, 7901, 7902, 7903, 7904, 7905,
+ 7906, 7907, 7908, 7909, 7910, 7911, 7912, 7913, 7914, 7915, 7916, 7917, 7918, 7919, 7920, 7921,
+ 7922, 7923, 7924, 7925, 7926, 7927, 7928, 7929, 7936, 7937, 7938, 7939, 7940, 7941, 7942, 7943,
+ 7944, 7945, 7946, 7947, 7948, 7949, 7950, 7951, 7952, 7953, 7954, 7955, 7956, 7957, 7960, 7961,
+ 7962, 7963, 7964, 7965, 7968, 7969, 7970, 7971, 7972, 7973, 7974, 7975, 7976, 7977, 7978, 7979,
+ 7980, 7981, 7982, 7983, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991, 7992, 7993, 7994, 7995,
+ 7996, 7997, 7998, 7999, 8000, 8001, 8002, 8003, 8004, 8005, 8008, 8009, 8010, 8011, 8012, 8013,
+ 8016, 8017, 8018, 8019, 8020, 8021, 8022, 8023, 8025, 8027, 8029, 8031, 8032, 8033, 8034, 8035,
+ 8036, 8037, 8038, 8039, 8040, 8041, 8042, 8043, 8044, 8045, 8046, 8047, 8048, 8049, 8050, 8051,
+ 8052, 8053, 8054, 8055, 8056, 8057, 8058, 8059, 8060, 8061, 8064, 8065, 8066, 8067, 8068, 8069,
+ 8070, 8071, 8072, 8073, 8074, 8075, 8076, 8077, 8078, 8079, 8080, 8081, 8082, 8083, 8084, 8085,
+ 8086, 8087, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8096, 8097, 8098, 8099, 8100, 8101,
+ 8102, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111, 8112, 8113, 8114, 8115, 8116, 8118,
+ 8119, 8120, 8121, 8122, 8123, 8124, 8126, 8129, 8130, 8131, 8132, 8134, 8135, 8136, 8137, 8138,
+ 8139, 8140, 8141, 8142, 8143, 8144, 8145, 8146, 8147, 8150, 8151, 8152, 8153, 8154, 8155, 8157,
+ 8158, 8159, 8160, 8161, 8162, 8163, 8164, 8165, 8166, 8167, 8168, 8169, 8170, 8171, 8172, 8173,
+ 8174, 8175, 8178, 8179, 8180, 8182, 8183, 8184, 8185, 8186, 8187, 8188, 8189, 8192, 8193, 8486,
+ 8490, 8491, 8602, 8603, 8622, 8653, 8654, 8655, 8708, 8713, 8716, 8740, 8742, 8769, 8772, 8775,
+ 8777, 8800, 8802, 8813, 8814, 8815, 8816, 8817, 8820, 8821, 8824, 8825, 8832, 8833, 8836, 8837,
+ 8840, 8841, 8876, 8877, 8878, 8879, 8928, 8929, 8930, 8931, 8938, 8939, 8940, 8941, 9001, 9002,
+ 10972, 12364, 12366, 12368, 12370, 12372, 12374, 12376, 12378, 12380, 12382, 12384, 12386,
+ 12389, 12391, 12393, 12400, 12401, 12403, 12404, 12406, 12407, 12409, 12410, 12412, 12413,
+ 12436, 12446, 12460, 12462, 12464, 12466, 12468, 12470, 12472, 12474, 12476, 12478, 12480,
+ 12482, 12485, 12487, 12489, 12496, 12497, 12499, 12500, 12502, 12503, 12505, 12506, 12508,
+ 12509, 12532, 12535, 12536, 12537, 12538, 12542, 63744, 63745, 63746, 63747, 63748, 63749,
+ 63750, 63751, 63752, 63753, 63754, 63755, 63756, 63757, 63758, 63759, 63760, 63761, 63762,
+ 63763, 63764, 63765, 63766, 63767, 63768, 63769, 63770, 63771, 63772, 63773, 63774, 63775,
+ 63776, 63777, 63778, 63779, 63780, 63781, 63782, 63783, 63784, 63785, 63786, 63787, 63788,
+ 63789, 63790, 63791, 63792, 63793, 63794, 63795, 63796, 63797, 63798, 63799, 63800, 63801,
+ 63802, 63803, 63804, 63805, 63806, 63807, 63808, 63809, 63810, 63811, 63812, 63813, 63814,
+ 63815, 63816, 63817, 63818, 63819, 63820, 63821, 63822, 63823, 63824, 63825, 63826, 63827,
+ 63828, 63829, 63830, 63831, 63832, 63833, 63834, 63835, 63836, 63837, 63838, 63839, 63840,
+ 63841, 63842, 63843, 63844, 63845, 63846, 63847, 63848, 63849, 63850, 63851, 63852, 63853,
+ 63854, 63855, 63856, 63857, 63858, 63859, 63860, 63861, 63862, 63863, 63864, 63865, 63866,
+ 63867, 63868, 63869, 63870, 63871, 63872, 63873, 63874, 63875, 63876, 63877, 63878, 63879,
+ 63880, 63881, 63882, 63883, 63884, 63885, 63886, 63887, 63888, 63889, 63890, 63891, 63892,
+ 63893, 63894, 63895, 63896, 63897, 63898, 63899, 63900, 63901, 63902, 63903, 63904, 63905,
+ 63906, 63907, 63908, 63909, 63910, 63911, 63912, 63913, 63914, 63915, 63916, 63917, 63918,
+ 63919, 63920, 63921, 63922, 63923, 63924, 63925, 63926, 63927, 63928, 63929, 63930, 63931,
+ 63932, 63933, 63934, 63935, 63936, 63937, 63938, 63939, 63940, 63941, 63942, 63943, 63944,
+ 63945, 63946, 63947, 63948, 63949, 63950, 63951, 63952, 63953, 63954, 63955, 63956, 63957,
+ 63958, 63959, 63960, 63961, 63962, 63963, 63964, 63965, 63966, 63967, 63968, 63969, 63970,
+ 63971, 63972, 63973, 63974, 63975, 63976, 63977, 63978, 63979, 63980, 63981, 63982, 63983,
+ 63984, 63985, 63986, 63987, 63988, 63989, 63990, 63991, 63992, 63993, 63994, 63995, 63996,
+ 63997, 63998, 63999, 64000, 64001, 64002, 64003, 64004, 64005, 64006, 64007, 64008, 64009,
+ 64010, 64011, 64012, 64013, 64016, 64018, 64021, 64022, 64023, 64024, 64025, 64026, 64027,
+ 64028, 64029, 64030, 64032, 64034, 64037, 64038, 64042, 64043, 64044, 64045, 64046, 64047,
+ 64048, 64049, 64050, 64051, 64052, 64053, 64054, 64055, 64056, 64057, 64058, 64059, 64060,
+ 64061, 64062, 64063, 64064, 64065, 64066, 64067, 64068, 64069, 64070, 64071, 64072, 64073,
+ 64074, 64075, 64076, 64077, 64078, 64079, 64080, 64081, 64082, 64083, 64084, 64085, 64086,
+ 64087, 64088, 64089, 64090, 64091, 64092, 64093, 64094, 64095, 64096, 64097, 64098, 64099,
+ 64100, 64101, 64102, 64103, 64104, 64105, 64106, 64107, 64108, 64109, 64112, 64113, 64114,
+ 64115, 64116, 64117, 64118, 64119, 64120, 64121, 64122, 64123, 64124, 64125, 64126, 64127,
+ 64128, 64129, 64130, 64131, 64132, 64133, 64134, 64135, 64136, 64137, 64138, 64139, 64140,
+ 64141, 64142, 64143, 64144, 64145, 64146, 64147, 64148, 64149, 64150, 64151, 64152, 64153,
+ 64154, 64155, 64156, 64157, 64158, 64159, 64160, 64161, 64162, 64163, 64164, 64165, 64166,
+ 64167, 64168, 64169, 64170, 64171, 64172, 64173, 64174, 64175, 64176, 64177, 64178, 64179,
+ 64180, 64181, 64182, 64183, 64184, 64185, 64186, 64187, 64188, 64189, 64190, 64191, 64192,
+ 64193, 64194, 64195, 64196, 64197, 64198, 64199, 64200, 64201, 64202, 64203, 64204, 64205,
+ 64206, 64207, 64208, 64209, 64210, 64211, 64212, 64213, 64214, 64215, 64216, 64217, 64285,
+ 64287, 64298, 64299, 64300, 64301, 64302, 64303, 64304, 64305, 64306, 64307, 64308, 64309,
+ 64310, 64312, 64313, 64314, 64315, 64316, 64318, 64320, 64321, 64323, 64324, 64326, 64327,
+ 64328, 64329, 64330, 64331, 64332, 64333, 64334, 69786, 69788, 69803, 69934, 69935, 70475,
+ 70476, 70843, 70844, 70846, 71098, 71099, 119134, 119135, 119136, 119137, 119138, 119139,
+ 119140, 119227, 119228, 119229, 119230, 119231, 119232, 194560, 194561, 194562, 194563, 194564,
+ 194565, 194566, 194567, 194568, 194569, 194570, 194571, 194572, 194573, 194574, 194575, 194576,
+ 194577, 194578, 194579, 194580, 194581, 194582, 194583, 194584, 194585, 194586, 194587, 194588,
+ 194589, 194590, 194591, 194592, 194593, 194594, 194595, 194596, 194597, 194598, 194599, 194600,
+ 194601, 194602, 194603, 194604, 194605, 194606, 194607, 194608, 194609, 194610, 194611, 194612,
+ 194613, 194614, 194615, 194616, 194617, 194618, 194619, 194620, 194621, 194622, 194623, 194624,
+ 194625, 194626, 194627, 194628, 194629, 194630, 194631, 194632, 194633, 194634, 194635, 194636,
+ 194637, 194638, 194639, 194640, 194641, 194642, 194643, 194644, 194645, 194646, 194647, 194648,
+ 194649, 194650, 194651, 194652, 194653, 194654, 194655, 194656, 194657, 194658, 194659, 194660,
+ 194661, 194662, 194663, 194664, 194665, 194666, 194667, 194668, 194669, 194670, 194671, 194672,
+ 194673, 194674, 194675, 194676, 194677, 194678, 194679, 194680, 194681, 194682, 194683, 194684,
+ 194685, 194686, 194687, 194688, 194689, 194690, 194691, 194692, 194693, 194694, 194695, 194696,
+ 194697, 194698, 194699, 194700, 194701, 194702, 194703, 194704, 194705, 194706, 194707, 194708,
+ 194709, 194710, 194711, 194712, 194713, 194714, 194715, 194716, 194717, 194718, 194719, 194720,
+ 194721, 194722, 194723, 194724, 194725, 194726, 194727, 194728, 194729, 194730, 194731, 194732,
+ 194733, 194734, 194735, 194736, 194737, 194738, 194739, 194740, 194741, 194742, 194743, 194744,
+ 194745, 194746, 194747, 194748, 194749, 194750, 194751, 194752, 194753, 194754, 194755, 194756,
+ 194757, 194758, 194759, 194760, 194761, 194762, 194763, 194764, 194765, 194766, 194767, 194768,
+ 194769, 194770, 194771, 194772, 194773, 194774, 194775, 194776, 194777, 194778, 194779, 194780,
+ 194781, 194782, 194783, 194784, 194785, 194786, 194787, 194788, 194789, 194790, 194791, 194792,
+ 194793, 194794, 194795, 194796, 194797, 194798, 194799, 194800, 194801, 194802, 194803, 194804,
+ 194805, 194806, 194807, 194808, 194809, 194810, 194811, 194812, 194813, 194814, 194815, 194816,
+ 194817, 194818, 194819, 194820, 194821, 194822, 194823, 194824, 194825, 194826, 194827, 194828,
+ 194829, 194830, 194831, 194832, 194833, 194834, 194835, 194836, 194837, 194838, 194839, 194840,
+ 194841, 194842, 194843, 194844, 194845, 194846, 194847, 194848, 194849, 194850, 194851, 194852,
+ 194853, 194854, 194855, 194856, 194857, 194858, 194859, 194860, 194861, 194862, 194863, 194864,
+ 194865, 194866, 194867, 194868, 194869, 194870, 194871, 194872, 194873, 194874, 194875, 194876,
+ 194877, 194878, 194879, 194880, 194881, 194882, 194883, 194884, 194885, 194886, 194887, 194888,
+ 194889, 194890, 194891, 194892, 194893, 194894, 194895, 194896, 194897, 194898, 194899, 194900,
+ 194901, 194902, 194903, 194904, 194905, 194906, 194907, 194908, 194909, 194910, 194911, 194912,
+ 194913, 194914, 194915, 194916, 194917, 194918, 194919, 194920, 194921, 194922, 194923, 194924,
+ 194925, 194926, 194927, 194928, 194929, 194930, 194931, 194932, 194933, 194934, 194935, 194936,
+ 194937, 194938, 194939, 194940, 194941, 194942, 194943, 194944, 194945, 194946, 194947, 194948,
+ 194949, 194950, 194951, 194952, 194953, 194954, 194955, 194956, 194957, 194958, 194959, 194960,
+ 194961, 194962, 194963, 194964, 194965, 194966, 194967, 194968, 194969, 194970, 194971, 194972,
+ 194973, 194974, 194975, 194976, 194977, 194978, 194979, 194980, 194981, 194982, 194983, 194984,
+ 194985, 194986, 194987, 194988, 194989, 194990, 194991, 194992, 194993, 194994, 194995, 194996,
+ 194997, 194998, 194999, 195000, 195001, 195002, 195003, 195004, 195005, 195006, 195007, 195008,
+ 195009, 195010, 195011, 195012, 195013, 195014, 195015, 195016, 195017, 195018, 195019, 195020,
+ 195021, 195022, 195023, 195024, 195025, 195026, 195027, 195028, 195029, 195030, 195031, 195032,
+ 195033, 195034, 195035, 195036, 195037, 195038, 195039, 195040, 195041, 195042, 195043, 195044,
+ 195045, 195046, 195047, 195048, 195049, 195050, 195051, 195052, 195053, 195054, 195055, 195056,
+ 195057, 195058, 195059, 195060, 195061, 195062, 195063, 195064, 195065, 195066, 195067, 195068,
+ 195069, 195070, 195071, 195072, 195073, 195074, 195075, 195076, 195077, 195078, 195079, 195080,
+ 195081, 195082, 195083, 195084, 195085, 195086, 195087, 195088, 195089, 195090, 195091, 195092,
+ 195093, 195094, 195095, 195096, 195097, 195098, 195099, 195100, 195101,
+};
+
+// offsets into k_canon_pool
+inline constexpr uint32_t k_canon_offsets[] = {
+ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48,
+ 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96,
+ 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134,
+ 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 160, 162, 164, 166, 168, 170, 172,
+ 174, 176, 178, 180, 182, 184, 186, 188, 190, 192, 194, 196, 198, 200, 202, 204, 206, 208, 210,
+ 212, 214, 216, 218, 220, 222, 224, 226, 228, 230, 232, 234, 236, 238, 240, 242, 244, 246, 248,
+ 250, 252, 254, 256, 258, 260, 262, 264, 266, 268, 270, 272, 274, 276, 278, 280, 282, 284, 286,
+ 288, 290, 292, 294, 296, 298, 300, 302, 304, 306, 308, 310, 312, 314, 316, 318, 320, 322, 324,
+ 326, 328, 330, 332, 334, 336, 338, 340, 342, 344, 346, 349, 352, 355, 358, 361, 364, 367, 370,
+ 373, 376, 379, 382, 384, 386, 388, 390, 392, 394, 396, 398, 401, 404, 406, 408, 410, 412, 414,
+ 416, 418, 421, 424, 426, 428, 430, 432, 434, 436, 438, 440, 442, 444, 446, 448, 450, 452, 454,
+ 456, 458, 460, 462, 464, 466, 468, 470, 472, 474, 476, 478, 480, 482, 484, 486, 488, 490, 492,
+ 494, 496, 498, 500, 503, 506, 509, 512, 514, 516, 519, 522, 524, 526, 527, 528, 529, 531, 532,
+ 533, 535, 537, 538, 540, 542, 544, 546, 548, 550, 553, 555, 557, 559, 561, 563, 565, 568, 570,
+ 572, 574, 576, 578, 580, 582, 584, 586, 588, 590, 592, 594, 596, 598, 600, 602, 604, 606, 608,
+ 610, 612, 614, 616, 618, 620, 622, 624, 626, 628, 630, 632, 634, 636, 638, 640, 642, 644, 646,
+ 648, 650, 652, 654, 656, 658, 660, 662, 664, 666, 668, 670, 672, 674, 676, 678, 680, 682, 684,
+ 686, 688, 690, 692, 694, 696, 698, 700, 702, 704, 706, 708, 710, 712, 714, 716, 718, 720, 722,
+ 724, 726, 728, 730, 732, 734, 736, 738, 740, 742, 744, 746, 748, 750, 752, 754, 756, 758, 760,
+ 762, 764, 766, 768, 770, 772, 774, 777, 779, 781, 783, 785, 787, 790, 792, 794, 796, 798, 800,
+ 802, 804, 806, 808, 810, 812, 814, 816, 818, 820, 822, 824, 826, 828, 830, 832, 834, 836, 838,
+ 840, 842, 844, 846, 848, 850, 852, 854, 856, 858, 860, 862, 864, 866, 869, 872, 874, 876, 878,
+ 880, 882, 884, 886, 888, 890, 892, 895, 898, 901, 904, 906, 908, 910, 912, 915, 918, 920, 922,
+ 924, 926, 928, 930, 932, 934, 936, 938, 940, 942, 944, 946, 948, 950, 953, 956, 958, 960, 962,
+ 964, 966, 968, 970, 972, 975, 978, 980, 982, 984, 986, 988, 990, 992, 994, 996, 998, 1000,
+ 1002, 1004, 1006, 1008, 1010, 1012, 1014, 1017, 1020, 1023, 1026, 1029, 1032, 1035, 1038, 1040,
+ 1042, 1044, 1046, 1048, 1050, 1052, 1054, 1057, 1060, 1062, 1064, 1066, 1068, 1070, 1072, 1075,
+ 1078, 1081, 1084, 1087, 1090, 1092, 1094, 1096, 1098, 1100, 1102, 1104, 1106, 1108, 1110, 1112,
+ 1114, 1116, 1118, 1121, 1124, 1127, 1130, 1132, 1134, 1136, 1138, 1140, 1142, 1144, 1146, 1148,
+ 1150, 1152, 1154, 1156, 1158, 1160, 1162, 1164, 1166, 1168, 1170, 1172, 1174, 1176, 1178, 1180,
+ 1182, 1184, 1186, 1188, 1190, 1192, 1194, 1196, 1198, 1200, 1203, 1206, 1209, 1212, 1215, 1218,
+ 1221, 1224, 1227, 1230, 1233, 1236, 1239, 1242, 1245, 1248, 1251, 1254, 1257, 1260, 1262, 1264,
+ 1266, 1268, 1270, 1272, 1275, 1278, 1281, 1284, 1287, 1290, 1293, 1296, 1299, 1302, 1304, 1306,
+ 1308, 1310, 1312, 1314, 1316, 1318, 1321, 1324, 1327, 1330, 1333, 1336, 1339, 1342, 1345, 1348,
+ 1351, 1354, 1357, 1360, 1363, 1366, 1369, 1372, 1375, 1378, 1380, 1382, 1384, 1386, 1389, 1392,
+ 1395, 1398, 1401, 1404, 1407, 1410, 1413, 1416, 1418, 1420, 1422, 1424, 1426, 1428, 1430, 1432,
+ 1434, 1436, 1439, 1442, 1445, 1448, 1451, 1454, 1456, 1458, 1461, 1464, 1467, 1470, 1473, 1476,
+ 1478, 1480, 1483, 1486, 1489, 1492, 1494, 1496, 1499, 1502, 1505, 1508, 1510, 1512, 1515, 1518,
+ 1521, 1524, 1527, 1530, 1532, 1534, 1537, 1540, 1543, 1546, 1549, 1552, 1554, 1556, 1559, 1562,
+ 1565, 1568, 1571, 1574, 1576, 1578, 1581, 1584, 1587, 1590, 1593, 1596, 1598, 1600, 1603, 1606,
+ 1609, 1612, 1614, 1616, 1619, 1622, 1625, 1628, 1630, 1632, 1635, 1638, 1641, 1644, 1647, 1650,
+ 1652, 1655, 1658, 1661, 1663, 1665, 1668, 1671, 1674, 1677, 1680, 1683, 1685, 1687, 1690, 1693,
+ 1696, 1699, 1702, 1705, 1707, 1709, 1711, 1713, 1715, 1717, 1719, 1721, 1723, 1725, 1727, 1729,
+ 1731, 1733, 1736, 1739, 1743, 1747, 1751, 1755, 1759, 1763, 1766, 1769, 1773, 1777, 1781, 1785,
+ 1789, 1793, 1796, 1799, 1803, 1807, 1811, 1815, 1819, 1823, 1826, 1829, 1833, 1837, 1841, 1845,
+ 1849, 1853, 1856, 1859, 1863, 1867, 1871, 1875, 1879, 1883, 1886, 1889, 1893, 1897, 1901, 1905,
+ 1909, 1913, 1915, 1917, 1920, 1922, 1925, 1927, 1930, 1932, 1934, 1936, 1938, 1940, 1941, 1943,
+ 1946, 1948, 1951, 1953, 1956, 1958, 1960, 1962, 1964, 1966, 1968, 1970, 1972, 1974, 1976, 1979,
+ 1982, 1984, 1987, 1989, 1991, 1993, 1995, 1997, 1999, 2001, 2003, 2005, 2008, 2011, 2013, 2015,
+ 2017, 2020, 2022, 2024, 2026, 2028, 2030, 2032, 2034, 2035, 2038, 2040, 2043, 2045, 2048, 2050,
+ 2052, 2054, 2056, 2058, 2059, 2060, 2061, 2062, 2063, 2065, 2067, 2069, 2071, 2073, 2075, 2077,
+ 2079, 2081, 2083, 2085, 2087, 2089, 2091, 2093, 2095, 2097, 2099, 2101, 2103, 2105, 2107, 2109,
+ 2111, 2113, 2115, 2117, 2119, 2121, 2123, 2125, 2127, 2129, 2131, 2133, 2135, 2137, 2139, 2141,
+ 2143, 2145, 2147, 2149, 2151, 2153, 2154, 2155, 2157, 2159, 2161, 2163, 2165, 2167, 2169, 2171,
+ 2173, 2175, 2177, 2179, 2181, 2183, 2185, 2187, 2189, 2191, 2193, 2195, 2197, 2199, 2201, 2203,
+ 2205, 2207, 2209, 2211, 2213, 2215, 2217, 2219, 2221, 2223, 2225, 2227, 2229, 2231, 2233, 2235,
+ 2237, 2239, 2241, 2243, 2245, 2247, 2249, 2251, 2253, 2255, 2257, 2259, 2261, 2263, 2265, 2267,
+ 2269, 2271, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286,
+ 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302,
+ 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318,
+ 2319, 2320, 2321, 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334,
+ 2335, 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350,
+ 2351, 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366,
+ 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382,
+ 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398,
+ 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414,
+ 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2429, 2430,
+ 2431, 2432, 2433, 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2446,
+ 2447, 2448, 2449, 2450, 2451, 2452, 2453, 2454, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462,
+ 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2476, 2477, 2478,
+ 2479, 2480, 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494,
+ 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510,
+ 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526,
+ 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542,
+ 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558,
+ 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2568, 2569, 2570, 2571, 2572, 2573, 2574,
+ 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590,
+ 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606,
+ 2607, 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622,
+ 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2638,
+ 2639, 2640, 2641, 2642, 2643, 2644, 2645, 2646, 2647, 2648, 2649, 2650, 2651, 2652, 2653, 2654,
+ 2655, 2656, 2657, 2658, 2659, 2660, 2661, 2662, 2663, 2664, 2665, 2666, 2667, 2668, 2669, 2670,
+ 2671, 2672, 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, 2684, 2685, 2686,
+ 2687, 2688, 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, 2700, 2701, 2702,
+ 2703, 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718,
+ 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2735,
+ 2737, 2739, 2741, 2744, 2747, 2749, 2751, 2753, 2755, 2757, 2759, 2761, 2763, 2765, 2767, 2769,
+ 2771, 2773, 2775, 2777, 2779, 2781, 2783, 2785, 2787, 2789, 2791, 2793, 2795, 2797, 2799, 2801,
+ 2803, 2805, 2807, 2809, 2811, 2813, 2815, 2817, 2819, 2821, 2823, 2825, 2827, 2829, 2831, 2834,
+ 2837, 2840, 2843, 2846, 2848, 2850, 2853, 2856, 2859, 2862, 2863, 2864, 2865, 2866, 2867, 2868,
+ 2869, 2870, 2871, 2872, 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, 2884,
+ 2885, 2886, 2887, 2888, 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, 2900,
+ 2901, 2902, 2903, 2904, 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, 2916,
+ 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931, 2932,
+ 2933, 2934, 2935, 2936, 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, 2948,
+ 2949, 2950, 2951, 2952, 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, 2964,
+ 2965, 2966, 2967, 2968, 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, 2980,
+ 2981, 2982, 2983, 2984, 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, 2996,
+ 2997, 2998, 2999, 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, 3012,
+ 3013, 3014, 3015, 3016, 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, 3028,
+ 3029, 3030, 3031, 3032, 3033, 3034, 3035, 3036, 3037, 3038, 3039, 3040, 3041, 3042, 3043, 3044,
+ 3045, 3046, 3047, 3048, 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, 3060,
+ 3061, 3062, 3063, 3064, 3065, 3066, 3067, 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, 3076,
+ 3077, 3078, 3079, 3080, 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, 3092,
+ 3093, 3094, 3095, 3096, 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108,
+ 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124,
+ 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, 3140,
+ 3141, 3142, 3143, 3144, 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, 3156,
+ 3157, 3158, 3159, 3160, 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, 3172,
+ 3173, 3174, 3175, 3176, 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, 3188,
+ 3189, 3190, 3191, 3192, 3193, 3194, 3195, 3196, 3197, 3198, 3199, 3200, 3201, 3202, 3203, 3204,
+ 3205, 3206, 3207, 3208, 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, 3220,
+ 3221, 3222, 3223, 3224, 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, 3236,
+ 3237, 3238, 3239, 3240, 3241, 3242, 3243, 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, 3252,
+ 3253, 3254, 3255, 3256, 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, 3268,
+ 3269, 3270, 3271, 3272, 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, 3284,
+ 3285, 3286, 3287, 3288, 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, 3300,
+ 3301, 3302, 3303, 3304, 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, 3316,
+ 3317, 3318, 3319, 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, 3332,
+ 3333, 3334, 3335, 3336, 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, 3348,
+ 3349, 3350, 3351, 3352, 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, 3364,
+ 3365, 3366, 3367, 3368, 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, 3380,
+ 3381, 3382, 3383, 3384, 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, 3396,
+ 3397, 3398, 3399, 3400, 3401, 3402, 3403, 3404,
+};
+
+// the decompositions of k_canon_keys
+inline constexpr uint32_t k_canon_pool[] = {
+ 65, 768, 65, 769, 65, 770, 65, 771, 65, 776, 65, 778, 67, 807, 69, 768, 69, 769, 69, 770, 69,
+ 776, 73, 768, 73, 769, 73, 770, 73, 776, 78, 771, 79, 768, 79, 769, 79, 770, 79, 771, 79, 776,
+ 85, 768, 85, 769, 85, 770, 85, 776, 89, 769, 97, 768, 97, 769, 97, 770, 97, 771, 97, 776, 97,
+ 778, 99, 807, 101, 768, 101, 769, 101, 770, 101, 776, 105, 768, 105, 769, 105, 770, 105, 776,
+ 110, 771, 111, 768, 111, 769, 111, 770, 111, 771, 111, 776, 117, 768, 117, 769, 117, 770, 117,
+ 776, 121, 769, 121, 776, 65, 772, 97, 772, 65, 774, 97, 774, 65, 808, 97, 808, 67, 769, 99,
+ 769, 67, 770, 99, 770, 67, 775, 99, 775, 67, 780, 99, 780, 68, 780, 100, 780, 69, 772, 101,
+ 772, 69, 774, 101, 774, 69, 775, 101, 775, 69, 808, 101, 808, 69, 780, 101, 780, 71, 770, 103,
+ 770, 71, 774, 103, 774, 71, 775, 103, 775, 71, 807, 103, 807, 72, 770, 104, 770, 73, 771, 105,
+ 771, 73, 772, 105, 772, 73, 774, 105, 774, 73, 808, 105, 808, 73, 775, 74, 770, 106, 770, 75,
+ 807, 107, 807, 76, 769, 108, 769, 76, 807, 108, 807, 76, 780, 108, 780, 78, 769, 110, 769, 78,
+ 807, 110, 807, 78, 780, 110, 780, 79, 772, 111, 772, 79, 774, 111, 774, 79, 779, 111, 779, 82,
+ 769, 114, 769, 82, 807, 114, 807, 82, 780, 114, 780, 83, 769, 115, 769, 83, 770, 115, 770, 83,
+ 807, 115, 807, 83, 780, 115, 780, 84, 807, 116, 807, 84, 780, 116, 780, 85, 771, 117, 771, 85,
+ 772, 117, 772, 85, 774, 117, 774, 85, 778, 117, 778, 85, 779, 117, 779, 85, 808, 117, 808, 87,
+ 770, 119, 770, 89, 770, 121, 770, 89, 776, 90, 769, 122, 769, 90, 775, 122, 775, 90, 780, 122,
+ 780, 79, 795, 111, 795, 85, 795, 117, 795, 65, 780, 97, 780, 73, 780, 105, 780, 79, 780, 111,
+ 780, 85, 780, 117, 780, 85, 776, 772, 117, 776, 772, 85, 776, 769, 117, 776, 769, 85, 776, 780,
+ 117, 776, 780, 85, 776, 768, 117, 776, 768, 65, 776, 772, 97, 776, 772, 65, 775, 772, 97, 775,
+ 772, 198, 772, 230, 772, 71, 780, 103, 780, 75, 780, 107, 780, 79, 808, 111, 808, 79, 808, 772,
+ 111, 808, 772, 439, 780, 658, 780, 106, 780, 71, 769, 103, 769, 78, 768, 110, 768, 65, 778,
+ 769, 97, 778, 769, 198, 769, 230, 769, 216, 769, 248, 769, 65, 783, 97, 783, 65, 785, 97, 785,
+ 69, 783, 101, 783, 69, 785, 101, 785, 73, 783, 105, 783, 73, 785, 105, 785, 79, 783, 111, 783,
+ 79, 785, 111, 785, 82, 783, 114, 783, 82, 785, 114, 785, 85, 783, 117, 783, 85, 785, 117, 785,
+ 83, 806, 115, 806, 84, 806, 116, 806, 72, 780, 104, 780, 65, 775, 97, 775, 69, 807, 101, 807,
+ 79, 776, 772, 111, 776, 772, 79, 771, 772, 111, 771, 772, 79, 775, 111, 775, 79, 775, 772, 111,
+ 775, 772, 89, 772, 121, 772, 768, 769, 787, 776, 769, 697, 59, 168, 769, 913, 769, 183, 917,
+ 769, 919, 769, 921, 769, 927, 769, 933, 769, 937, 769, 953, 776, 769, 921, 776, 933, 776, 945,
+ 769, 949, 769, 951, 769, 953, 769, 965, 776, 769, 953, 776, 965, 776, 959, 769, 965, 769, 969,
+ 769, 978, 769, 978, 776, 1045, 768, 1045, 776, 1043, 769, 1030, 776, 1050, 769, 1048, 768,
+ 1059, 774, 1048, 774, 1080, 774, 1077, 768, 1077, 776, 1075, 769, 1110, 776, 1082, 769, 1080,
+ 768, 1091, 774, 1140, 783, 1141, 783, 1046, 774, 1078, 774, 1040, 774, 1072, 774, 1040, 776,
+ 1072, 776, 1045, 774, 1077, 774, 1240, 776, 1241, 776, 1046, 776, 1078, 776, 1047, 776, 1079,
+ 776, 1048, 772, 1080, 772, 1048, 776, 1080, 776, 1054, 776, 1086, 776, 1256, 776, 1257, 776,
+ 1069, 776, 1101, 776, 1059, 772, 1091, 772, 1059, 776, 1091, 776, 1059, 779, 1091, 779, 1063,
+ 776, 1095, 776, 1067, 776, 1099, 776, 1575, 1619, 1575, 1620, 1608, 1620, 1575, 1621, 1610,
+ 1620, 1749, 1620, 1729, 1620, 1746, 1620, 2344, 2364, 2352, 2364, 2355, 2364, 2325, 2364, 2326,
+ 2364, 2327, 2364, 2332, 2364, 2337, 2364, 2338, 2364, 2347, 2364, 2351, 2364, 2503, 2494, 2503,
+ 2519, 2465, 2492, 2466, 2492, 2479, 2492, 2610, 2620, 2616, 2620, 2582, 2620, 2583, 2620, 2588,
+ 2620, 2603, 2620, 2887, 2902, 2887, 2878, 2887, 2903, 2849, 2876, 2850, 2876, 2962, 3031, 3014,
+ 3006, 3015, 3006, 3014, 3031, 3142, 3158, 3263, 3285, 3270, 3285, 3270, 3286, 3270, 3266, 3270,
+ 3266, 3285, 3398, 3390, 3399, 3390, 3398, 3415, 3545, 3530, 3545, 3535, 3545, 3535, 3530, 3545,
+ 3551, 3906, 4023, 3916, 4023, 3921, 4023, 3926, 4023, 3931, 4023, 3904, 4021, 3953, 3954, 3953,
+ 3956, 4018, 3968, 4019, 3968, 3953, 3968, 3986, 4023, 3996, 4023, 4001, 4023, 4006, 4023, 4011,
+ 4023, 3984, 4021, 4133, 4142, 6917, 6965, 6919, 6965, 6921, 6965, 6923, 6965, 6925, 6965, 6929,
+ 6965, 6970, 6965, 6972, 6965, 6974, 6965, 6975, 6965, 6978, 6965, 65, 805, 97, 805, 66, 775,
+ 98, 775, 66, 803, 98, 803, 66, 817, 98, 817, 67, 807, 769, 99, 807, 769, 68, 775, 100, 775, 68,
+ 803, 100, 803, 68, 817, 100, 817, 68, 807, 100, 807, 68, 813, 100, 813, 69, 772, 768, 101, 772,
+ 768, 69, 772, 769, 101, 772, 769, 69, 813, 101, 813, 69, 816, 101, 816, 69, 807, 774, 101, 807,
+ 774, 70, 775, 102, 775, 71, 772, 103, 772, 72, 775, 104, 775, 72, 803, 104, 803, 72, 776, 104,
+ 776, 72, 807, 104, 807, 72, 814, 104, 814, 73, 816, 105, 816, 73, 776, 769, 105, 776, 769, 75,
+ 769, 107, 769, 75, 803, 107, 803, 75, 817, 107, 817, 76, 803, 108, 803, 76, 803, 772, 108, 803,
+ 772, 76, 817, 108, 817, 76, 813, 108, 813, 77, 769, 109, 769, 77, 775, 109, 775, 77, 803, 109,
+ 803, 78, 775, 110, 775, 78, 803, 110, 803, 78, 817, 110, 817, 78, 813, 110, 813, 79, 771, 769,
+ 111, 771, 769, 79, 771, 776, 111, 771, 776, 79, 772, 768, 111, 772, 768, 79, 772, 769, 111,
+ 772, 769, 80, 769, 112, 769, 80, 775, 112, 775, 82, 775, 114, 775, 82, 803, 114, 803, 82, 803,
+ 772, 114, 803, 772, 82, 817, 114, 817, 83, 775, 115, 775, 83, 803, 115, 803, 83, 769, 775, 115,
+ 769, 775, 83, 780, 775, 115, 780, 775, 83, 803, 775, 115, 803, 775, 84, 775, 116, 775, 84, 803,
+ 116, 803, 84, 817, 116, 817, 84, 813, 116, 813, 85, 804, 117, 804, 85, 816, 117, 816, 85, 813,
+ 117, 813, 85, 771, 769, 117, 771, 769, 85, 772, 776, 117, 772, 776, 86, 771, 118, 771, 86, 803,
+ 118, 803, 87, 768, 119, 768, 87, 769, 119, 769, 87, 776, 119, 776, 87, 775, 119, 775, 87, 803,
+ 119, 803, 88, 775, 120, 775, 88, 776, 120, 776, 89, 775, 121, 775, 90, 770, 122, 770, 90, 803,
+ 122, 803, 90, 817, 122, 817, 104, 817, 116, 776, 119, 778, 121, 778, 383, 775, 65, 803, 97,
+ 803, 65, 777, 97, 777, 65, 770, 769, 97, 770, 769, 65, 770, 768, 97, 770, 768, 65, 770, 777,
+ 97, 770, 777, 65, 770, 771, 97, 770, 771, 65, 803, 770, 97, 803, 770, 65, 774, 769, 97, 774,
+ 769, 65, 774, 768, 97, 774, 768, 65, 774, 777, 97, 774, 777, 65, 774, 771, 97, 774, 771, 65,
+ 803, 774, 97, 803, 774, 69, 803, 101, 803, 69, 777, 101, 777, 69, 771, 101, 771, 69, 770, 769,
+ 101, 770, 769, 69, 770, 768, 101, 770, 768, 69, 770, 777, 101, 770, 777, 69, 770, 771, 101,
+ 770, 771, 69, 803, 770, 101, 803, 770, 73, 777, 105, 777, 73, 803, 105, 803, 79, 803, 111, 803,
+ 79, 777, 111, 777, 79, 770, 769, 111, 770, 769, 79, 770, 768, 111, 770, 768, 79, 770, 777, 111,
+ 770, 777, 79, 770, 771, 111, 770, 771, 79, 803, 770, 111, 803, 770, 79, 795, 769, 111, 795,
+ 769, 79, 795, 768, 111, 795, 768, 79, 795, 777, 111, 795, 777, 79, 795, 771, 111, 795, 771, 79,
+ 795, 803, 111, 795, 803, 85, 803, 117, 803, 85, 777, 117, 777, 85, 795, 769, 117, 795, 769, 85,
+ 795, 768, 117, 795, 768, 85, 795, 777, 117, 795, 777, 85, 795, 771, 117, 795, 771, 85, 795,
+ 803, 117, 795, 803, 89, 768, 121, 768, 89, 803, 121, 803, 89, 777, 121, 777, 89, 771, 121, 771,
+ 945, 787, 945, 788, 945, 787, 768, 945, 788, 768, 945, 787, 769, 945, 788, 769, 945, 787, 834,
+ 945, 788, 834, 913, 787, 913, 788, 913, 787, 768, 913, 788, 768, 913, 787, 769, 913, 788, 769,
+ 913, 787, 834, 913, 788, 834, 949, 787, 949, 788, 949, 787, 768, 949, 788, 768, 949, 787, 769,
+ 949, 788, 769, 917, 787, 917, 788, 917, 787, 768, 917, 788, 768, 917, 787, 769, 917, 788, 769,
+ 951, 787, 951, 788, 951, 787, 768, 951, 788, 768, 951, 787, 769, 951, 788, 769, 951, 787, 834,
+ 951, 788, 834, 919, 787, 919, 788, 919, 787, 768, 919, 788, 768, 919, 787, 769, 919, 788, 769,
+ 919, 787, 834, 919, 788, 834, 953, 787, 953, 788, 953, 787, 768, 953, 788, 768, 953, 787, 769,
+ 953, 788, 769, 953, 787, 834, 953, 788, 834, 921, 787, 921, 788, 921, 787, 768, 921, 788, 768,
+ 921, 787, 769, 921, 788, 769, 921, 787, 834, 921, 788, 834, 959, 787, 959, 788, 959, 787, 768,
+ 959, 788, 768, 959, 787, 769, 959, 788, 769, 927, 787, 927, 788, 927, 787, 768, 927, 788, 768,
+ 927, 787, 769, 927, 788, 769, 965, 787, 965, 788, 965, 787, 768, 965, 788, 768, 965, 787, 769,
+ 965, 788, 769, 965, 787, 834, 965, 788, 834, 933, 788, 933, 788, 768, 933, 788, 769, 933, 788,
+ 834, 969, 787, 969, 788, 969, 787, 768, 969, 788, 768, 969, 787, 769, 969, 788, 769, 969, 787,
+ 834, 969, 788, 834, 937, 787, 937, 788, 937, 787, 768, 937, 788, 768, 937, 787, 769, 937, 788,
+ 769, 937, 787, 834, 937, 788, 834, 945, 768, 945, 769, 949, 768, 949, 769, 951, 768, 951, 769,
+ 953, 768, 953, 769, 959, 768, 959, 769, 965, 768, 965, 769, 969, 768, 969, 769, 945, 787, 837,
+ 945, 788, 837, 945, 787, 768, 837, 945, 788, 768, 837, 945, 787, 769, 837, 945, 788, 769, 837,
+ 945, 787, 834, 837, 945, 788, 834, 837, 913, 787, 837, 913, 788, 837, 913, 787, 768, 837, 913,
+ 788, 768, 837, 913, 787, 769, 837, 913, 788, 769, 837, 913, 787, 834, 837, 913, 788, 834, 837,
+ 951, 787, 837, 951, 788, 837, 951, 787, 768, 837, 951, 788, 768, 837, 951, 787, 769, 837, 951,
+ 788, 769, 837, 951, 787, 834, 837, 951, 788, 834, 837, 919, 787, 837, 919, 788, 837, 919, 787,
+ 768, 837, 919, 788, 768, 837, 919, 787, 769, 837, 919, 788, 769, 837, 919, 787, 834, 837, 919,
+ 788, 834, 837, 969, 787, 837, 969, 788, 837, 969, 787, 768, 837, 969, 788, 768, 837, 969, 787,
+ 769, 837, 969, 788, 769, 837, 969, 787, 834, 837, 969, 788, 834, 837, 937, 787, 837, 937, 788,
+ 837, 937, 787, 768, 837, 937, 788, 768, 837, 937, 787, 769, 837, 937, 788, 769, 837, 937, 787,
+ 834, 837, 937, 788, 834, 837, 945, 774, 945, 772, 945, 768, 837, 945, 837, 945, 769, 837, 945,
+ 834, 945, 834, 837, 913, 774, 913, 772, 913, 768, 913, 769, 913, 837, 953, 168, 834, 951, 768,
+ 837, 951, 837, 951, 769, 837, 951, 834, 951, 834, 837, 917, 768, 917, 769, 919, 768, 919, 769,
+ 919, 837, 8127, 768, 8127, 769, 8127, 834, 953, 774, 953, 772, 953, 776, 768, 953, 776, 769,
+ 953, 834, 953, 776, 834, 921, 774, 921, 772, 921, 768, 921, 769, 8190, 768, 8190, 769, 8190,
+ 834, 965, 774, 965, 772, 965, 776, 768, 965, 776, 769, 961, 787, 961, 788, 965, 834, 965, 776,
+ 834, 933, 774, 933, 772, 933, 768, 933, 769, 929, 788, 168, 768, 168, 769, 96, 969, 768, 837,
+ 969, 837, 969, 769, 837, 969, 834, 969, 834, 837, 927, 768, 927, 769, 937, 768, 937, 769, 937,
+ 837, 180, 8194, 8195, 937, 75, 65, 778, 8592, 824, 8594, 824, 8596, 824, 8656, 824, 8660, 824,
+ 8658, 824, 8707, 824, 8712, 824, 8715, 824, 8739, 824, 8741, 824, 8764, 824, 8771, 824, 8773,
+ 824, 8776, 824, 61, 824, 8801, 824, 8781, 824, 60, 824, 62, 824, 8804, 824, 8805, 824, 8818,
+ 824, 8819, 824, 8822, 824, 8823, 824, 8826, 824, 8827, 824, 8834, 824, 8835, 824, 8838, 824,
+ 8839, 824, 8866, 824, 8872, 824, 8873, 824, 8875, 824, 8828, 824, 8829, 824, 8849, 824, 8850,
+ 824, 8882, 824, 8883, 824, 8884, 824, 8885, 824, 12296, 12297, 10973, 824, 12363, 12441, 12365,
+ 12441, 12367, 12441, 12369, 12441, 12371, 12441, 12373, 12441, 12375, 12441, 12377, 12441,
+ 12379, 12441, 12381, 12441, 12383, 12441, 12385, 12441, 12388, 12441, 12390, 12441, 12392,
+ 12441, 12399, 12441, 12399, 12442, 12402, 12441, 12402, 12442, 12405, 12441, 12405, 12442,
+ 12408, 12441, 12408, 12442, 12411, 12441, 12411, 12442, 12358, 12441, 12445, 12441, 12459,
+ 12441, 12461, 12441, 12463, 12441, 12465, 12441, 12467, 12441, 12469, 12441, 12471, 12441,
+ 12473, 12441, 12475, 12441, 12477, 12441, 12479, 12441, 12481, 12441, 12484, 12441, 12486,
+ 12441, 12488, 12441, 12495, 12441, 12495, 12442, 12498, 12441, 12498, 12442, 12501, 12441,
+ 12501, 12442, 12504, 12441, 12504, 12442, 12507, 12441, 12507, 12442, 12454, 12441, 12527,
+ 12441, 12528, 12441, 12529, 12441, 12530, 12441, 12541, 12441, 35912, 26356, 36554, 36040,
+ 28369, 20018, 21477, 40860, 40860, 22865, 37329, 21895, 22856, 25078, 30313, 32645, 34367,
+ 34746, 35064, 37007, 27138, 27931, 28889, 29662, 33853, 37226, 39409, 20098, 21365, 27396,
+ 29211, 34349, 40478, 23888, 28651, 34253, 35172, 25289, 33240, 34847, 24266, 26391, 28010,
+ 29436, 37070, 20358, 20919, 21214, 25796, 27347, 29200, 30439, 32769, 34310, 34396, 36335,
+ 38706, 39791, 40442, 30860, 31103, 32160, 33737, 37636, 40575, 35542, 22751, 24324, 31840,
+ 32894, 29282, 30922, 36034, 38647, 22744, 23650, 27155, 28122, 28431, 32047, 32311, 38475,
+ 21202, 32907, 20956, 20940, 31260, 32190, 33777, 38517, 35712, 25295, 27138, 35582, 20025,
+ 23527, 24594, 29575, 30064, 21271, 30971, 20415, 24489, 19981, 27852, 25976, 32034, 21443,
+ 22622, 30465, 33865, 35498, 27578, 36784, 27784, 25342, 33509, 25504, 30053, 20142, 20841,
+ 20937, 26753, 31975, 33391, 35538, 37327, 21237, 21570, 22899, 24300, 26053, 28670, 31018,
+ 38317, 39530, 40599, 40654, 21147, 26310, 27511, 36706, 24180, 24976, 25088, 25754, 28451,
+ 29001, 29833, 31178, 32244, 32879, 36646, 34030, 36899, 37706, 21015, 21155, 21693, 28872,
+ 35010, 35498, 24265, 24565, 25467, 27566, 31806, 29557, 20196, 22265, 23527, 23994, 24604,
+ 29618, 29801, 32666, 32838, 37428, 38646, 38728, 38936, 20363, 31150, 37300, 38584, 24801,
+ 20102, 20698, 23534, 23615, 26009, 27138, 29134, 30274, 34044, 36988, 40845, 26248, 38446,
+ 21129, 26491, 26611, 27969, 28316, 29705, 30041, 30827, 32016, 39006, 20845, 25134, 38520,
+ 20523, 23833, 28138, 36650, 24459, 24900, 26647, 29575, 38534, 21033, 21519, 23653, 26131,
+ 26446, 26792, 27877, 29702, 30178, 32633, 35023, 35041, 37324, 38626, 21311, 28346, 21533,
+ 29136, 29848, 34298, 38563, 40023, 40607, 26519, 28107, 33256, 31435, 31520, 31890, 29376,
+ 28825, 35672, 20160, 33590, 21050, 20999, 24230, 25299, 31958, 23429, 27934, 26292, 36667,
+ 34892, 38477, 35211, 24275, 20800, 21952, 22618, 26228, 20958, 29482, 30410, 31036, 31070,
+ 31077, 31119, 38742, 31934, 32701, 34322, 35576, 36920, 37117, 39151, 39164, 39208, 40372,
+ 37086, 38583, 20398, 20711, 20813, 21193, 21220, 21329, 21917, 22022, 22120, 22592, 22696,
+ 23652, 23662, 24724, 24936, 24974, 25074, 25935, 26082, 26257, 26757, 28023, 28186, 28450,
+ 29038, 29227, 29730, 30865, 31038, 31049, 31048, 31056, 31062, 31069, 31117, 31118, 31296,
+ 31361, 31680, 32244, 32265, 32321, 32626, 32773, 33261, 33401, 33401, 33879, 35088, 35222,
+ 35585, 35641, 36051, 36104, 36790, 36920, 38627, 38911, 38971, 24693, 148206, 33304, 20006,
+ 20917, 20840, 20352, 20805, 20864, 21191, 21242, 21917, 21845, 21913, 21986, 22618, 22707,
+ 22852, 22868, 23138, 23336, 24274, 24281, 24425, 24493, 24792, 24910, 24840, 24974, 24928,
+ 25074, 25140, 25540, 25628, 25682, 25942, 26228, 26391, 26395, 26454, 27513, 27578, 27969,
+ 28379, 28363, 28450, 28702, 29038, 30631, 29237, 29359, 29482, 29809, 29958, 30011, 30237,
+ 30239, 30410, 30427, 30452, 30538, 30528, 30924, 31409, 31680, 31867, 32091, 32244, 32574,
+ 32773, 33618, 33775, 34681, 35137, 35206, 35222, 35519, 35576, 35531, 35585, 35582, 35565,
+ 35641, 35722, 36104, 36664, 36978, 37273, 37494, 38524, 38627, 38742, 38875, 38911, 38923,
+ 38971, 39698, 40860, 141386, 141380, 144341, 15261, 16408, 16441, 152137, 154832, 163539,
+ 40771, 40846, 1497, 1460, 1522, 1463, 1513, 1473, 1513, 1474, 1513, 1468, 1473, 1513, 1468,
+ 1474, 1488, 1463, 1488, 1464, 1488, 1468, 1489, 1468, 1490, 1468, 1491, 1468, 1492, 1468, 1493,
+ 1468, 1494, 1468, 1496, 1468, 1497, 1468, 1498, 1468, 1499, 1468, 1500, 1468, 1502, 1468, 1504,
+ 1468, 1505, 1468, 1507, 1468, 1508, 1468, 1510, 1468, 1511, 1468, 1512, 1468, 1513, 1468, 1514,
+ 1468, 1493, 1465, 1489, 1471, 1499, 1471, 1508, 1471, 69785, 69818, 69787, 69818, 69797, 69818,
+ 69937, 69927, 69938, 69927, 70471, 70462, 70471, 70487, 70841, 70842, 70841, 70832, 70841,
+ 70845, 71096, 71087, 71097, 71087, 119127, 119141, 119128, 119141, 119128, 119141, 119150,
+ 119128, 119141, 119151, 119128, 119141, 119152, 119128, 119141, 119153, 119128, 119141, 119154,
+ 119225, 119141, 119226, 119141, 119225, 119141, 119150, 119226, 119141, 119150, 119225, 119141,
+ 119151, 119226, 119141, 119151, 20029, 20024, 20033, 131362, 20320, 20398, 20411, 20482, 20602,
+ 20633, 20711, 20687, 13470, 132666, 20813, 20820, 20836, 20855, 132380, 13497, 20839, 20877,
+ 132427, 20887, 20900, 20172, 20908, 20917, 168415, 20981, 20995, 13535, 21051, 21062, 21106,
+ 21111, 13589, 21191, 21193, 21220, 21242, 21253, 21254, 21271, 21321, 21329, 21338, 21363,
+ 21373, 21375, 21375, 21375, 133676, 28784, 21450, 21471, 133987, 21483, 21489, 21510, 21662,
+ 21560, 21576, 21608, 21666, 21750, 21776, 21843, 21859, 21892, 21892, 21913, 21931, 21939,
+ 21954, 22294, 22022, 22295, 22097, 22132, 20999, 22766, 22478, 22516, 22541, 22411, 22578,
+ 22577, 22700, 136420, 22770, 22775, 22790, 22810, 22818, 22882, 136872, 136938, 23020, 23067,
+ 23079, 23000, 23142, 14062, 14076, 23304, 23358, 23358, 137672, 23491, 23512, 23527, 23539,
+ 138008, 23551, 23558, 24403, 23586, 14209, 23648, 23662, 23744, 23693, 138724, 23875, 138726,
+ 23918, 23915, 23932, 24033, 24034, 14383, 24061, 24104, 24125, 24169, 14434, 139651, 14460,
+ 24240, 24243, 24246, 24266, 172946, 24318, 140081, 140081, 33281, 24354, 24354, 14535, 144056,
+ 156122, 24418, 24427, 14563, 24474, 24525, 24535, 24569, 24705, 14650, 14620, 24724, 141012,
+ 24775, 24904, 24908, 24910, 24908, 24954, 24974, 25010, 24996, 25007, 25054, 25074, 25078,
+ 25104, 25115, 25181, 25265, 25300, 25424, 142092, 25405, 25340, 25448, 25475, 25572, 142321,
+ 25634, 25541, 25513, 14894, 25705, 25726, 25757, 25719, 14956, 25935, 25964, 143370, 26083,
+ 26360, 26185, 15129, 26257, 15112, 15076, 20882, 20885, 26368, 26268, 32941, 17369, 26391,
+ 26395, 26401, 26462, 26451, 144323, 15177, 26618, 26501, 26706, 26757, 144493, 26766, 26655,
+ 26900, 15261, 26946, 27043, 27114, 27304, 145059, 27355, 15384, 27425, 145575, 27476, 15438,
+ 27506, 27551, 27578, 27579, 146061, 138507, 146170, 27726, 146620, 27839, 27853, 27751, 27926,
+ 27966, 28023, 27969, 28009, 28024, 28037, 146718, 27956, 28207, 28270, 15667, 28363, 28359,
+ 147153, 28153, 28526, 147294, 147342, 28614, 28729, 28702, 28699, 15766, 28746, 28797, 28791,
+ 28845, 132389, 28997, 148067, 29084, 148395, 29224, 29237, 29264, 149000, 29312, 29333, 149301,
+ 149524, 29562, 29579, 16044, 29605, 16056, 16056, 29767, 29788, 29809, 29829, 29898, 16155,
+ 29988, 150582, 30014, 150674, 30064, 139679, 30224, 151457, 151480, 151620, 16380, 16392,
+ 30452, 151795, 151794, 151833, 151859, 30494, 30495, 30495, 30538, 16441, 30603, 16454, 16534,
+ 152605, 30798, 30860, 30924, 16611, 153126, 31062, 153242, 153285, 31119, 31211, 16687, 31296,
+ 31306, 31311, 153980, 154279, 154279, 31470, 16898, 154539, 31686, 31689, 16935, 154752, 31954,
+ 17056, 31976, 31971, 32000, 155526, 32099, 17153, 32199, 32258, 32325, 17204, 156200, 156231,
+ 17241, 156377, 32634, 156478, 32661, 32762, 32773, 156890, 156963, 32864, 157096, 32880,
+ 144223, 17365, 32946, 33027, 17419, 33086, 23221, 157607, 157621, 144275, 144284, 33281, 33284,
+ 36766, 17515, 33425, 33419, 33437, 21171, 33457, 33459, 33469, 33510, 158524, 33509, 33565,
+ 33635, 33709, 33571, 33725, 33767, 33879, 33619, 33738, 33740, 33756, 158774, 159083, 158933,
+ 17707, 34033, 34035, 34070, 160714, 34148, 159532, 17757, 17761, 159665, 159954, 17771, 34384,
+ 34396, 34407, 34409, 34473, 34440, 34574, 34530, 34681, 34600, 34667, 34694, 17879, 34785,
+ 34817, 17913, 34912, 34915, 161383, 35031, 35038, 17973, 35066, 13499, 161966, 162150, 18110,
+ 18119, 35488, 35565, 35722, 35925, 162984, 36011, 36033, 36123, 36215, 163631, 133124, 36299,
+ 36284, 36336, 133342, 36564, 36664, 165330, 165357, 37012, 37105, 37137, 165678, 37147, 37432,
+ 37591, 37592, 37500, 37881, 37909, 166906, 38283, 18837, 38327, 167287, 18918, 38595, 23986,
+ 38691, 168261, 168474, 19054, 19062, 38880, 168970, 19122, 169110, 38923, 38923, 38953, 169398,
+ 39138, 19251, 39209, 39335, 39362, 39422, 19406, 170800, 39698, 40000, 40189, 19662, 19693,
+ 40295, 172238, 19704, 172293, 172558, 172689, 40635, 19798, 40697, 40702, 40709, 40719, 40726,
+ 40763, 173568,
+};
+
+// compat* hold the full compatibility decomposition (NFKD) of every code
+// point where it differs from the canonical one.
+inline constexpr uint32_t k_compat_keys[] = {
+ 160, 168, 170, 175, 178, 179, 180, 181, 184, 185, 186, 188, 189, 190, 306, 307, 319, 320, 329,
+ 383, 452, 453, 454, 455, 456, 457, 458, 459, 460, 497, 498, 499, 688, 689, 690, 691, 692, 693,
+ 694, 695, 696, 728, 729, 730, 731, 732, 733, 736, 737, 738, 739, 740, 890, 900, 901, 976, 977,
+ 978, 979, 980, 981, 982, 1008, 1009, 1010, 1012, 1013, 1017, 1415, 1653, 1654, 1655, 1656,
+ 3635, 3763, 3804, 3805, 3852, 3959, 3961, 4348, 7468, 7469, 7470, 7472, 7473, 7474, 7475, 7476,
+ 7477, 7478, 7479, 7480, 7481, 7482, 7484, 7485, 7486, 7487, 7488, 7489, 7490, 7491, 7492, 7493,
+ 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510,
+ 7511, 7512, 7513, 7514, 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, 7526,
+ 7527, 7528, 7529, 7530, 7544, 7579, 7580, 7581, 7582, 7583, 7584, 7585, 7586, 7587, 7588, 7589,
+ 7590, 7591, 7592, 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, 7605,
+ 7606, 7607, 7608, 7609, 7610, 7611, 7612, 7613, 7614, 7615, 7834, 7835, 8125, 8127, 8128, 8129,
+ 8141, 8142, 8143, 8157, 8158, 8159, 8173, 8174, 8189, 8190, 8192, 8193, 8194, 8195, 8196, 8197,
+ 8198, 8199, 8200, 8201, 8202, 8209, 8215, 8228, 8229, 8230, 8239, 8243, 8244, 8246, 8247, 8252,
+ 8254, 8263, 8264, 8265, 8279, 8287, 8304, 8305, 8308, 8309, 8310, 8311, 8312, 8313, 8314, 8315,
+ 8316, 8317, 8318, 8319, 8320, 8321, 8322, 8323, 8324, 8325, 8326, 8327, 8328, 8329, 8330, 8331,
+ 8332, 8333, 8334, 8336, 8337, 8338, 8339, 8340, 8341, 8342, 8343, 8344, 8345, 8346, 8347, 8348,
+ 8360, 8448, 8449, 8450, 8451, 8453, 8454, 8455, 8457, 8458, 8459, 8460, 8461, 8462, 8463, 8464,
+ 8465, 8466, 8467, 8469, 8470, 8473, 8474, 8475, 8476, 8477, 8480, 8481, 8482, 8484, 8488, 8492,
+ 8493, 8495, 8496, 8497, 8499, 8500, 8501, 8502, 8503, 8504, 8505, 8507, 8508, 8509, 8510, 8511,
+ 8512, 8517, 8518, 8519, 8520, 8521, 8528, 8529, 8530, 8531, 8532, 8533, 8534, 8535, 8536, 8537,
+ 8538, 8539, 8540, 8541, 8542, 8543, 8544, 8545, 8546, 8547, 8548, 8549, 8550, 8551, 8552, 8553,
+ 8554, 8555, 8556, 8557, 8558, 8559, 8560, 8561, 8562, 8563, 8564, 8565, 8566, 8567, 8568, 8569,
+ 8570, 8571, 8572, 8573, 8574, 8575, 8585, 8748, 8749, 8751, 8752, 9312, 9313, 9314, 9315, 9316,
+ 9317, 9318, 9319, 9320, 9321, 9322, 9323, 9324, 9325, 9326, 9327, 9328, 9329, 9330, 9331, 9332,
+ 9333, 9334, 9335, 9336, 9337, 9338, 9339, 9340, 9341, 9342, 9343, 9344, 9345, 9346, 9347, 9348,
+ 9349, 9350, 9351, 9352, 9353, 9354, 9355, 9356, 9357, 9358, 9359, 9360, 9361, 9362, 9363, 9364,
+ 9365, 9366, 9367, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380,
+ 9381, 9382, 9383, 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, 9393, 9394, 9395, 9396,
+ 9397, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410, 9411, 9412,
+ 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, 9424, 9425, 9426, 9427, 9428,
+ 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444,
+ 9445, 9446, 9447, 9448, 9449, 9450, 10764, 10868, 10869, 10870, 11388, 11389, 11631, 11935,
+ 12019, 12032, 12033, 12034, 12035, 12036, 12037, 12038, 12039, 12040, 12041, 12042, 12043,
+ 12044, 12045, 12046, 12047, 12048, 12049, 12050, 12051, 12052, 12053, 12054, 12055, 12056,
+ 12057, 12058, 12059, 12060, 12061, 12062, 12063, 12064, 12065, 12066, 12067, 12068, 12069,
+ 12070, 12071, 12072, 12073, 12074, 12075, 12076, 12077, 12078, 12079, 12080, 12081, 12082,
+ 12083, 12084, 12085, 12086, 12087, 12088, 12089, 12090, 12091, 12092, 12093, 12094, 12095,
+ 12096, 12097, 12098, 12099, 12100, 12101, 12102, 12103, 12104, 12105, 12106, 12107, 12108,
+ 12109, 12110, 12111, 12112, 12113, 12114, 12115, 12116, 12117, 12118, 12119, 12120, 12121,
+ 12122, 12123, 12124, 12125, 12126, 12127, 12128, 12129, 12130, 12131, 12132, 12133, 12134,
+ 12135, 12136, 12137, 12138, 12139, 12140, 12141, 12142, 12143, 12144, 12145, 12146, 12147,
+ 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155, 12156, 12157, 12158, 12159, 12160,
+ 12161, 12162, 12163, 12164, 12165, 12166, 12167, 12168, 12169, 12170, 12171, 12172, 12173,
+ 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, 12184, 12185, 12186,
+ 12187, 12188, 12189, 12190, 12191, 12192, 12193, 12194, 12195, 12196, 12197, 12198, 12199,
+ 12200, 12201, 12202, 12203, 12204, 12205, 12206, 12207, 12208, 12209, 12210, 12211, 12212,
+ 12213, 12214, 12215, 12216, 12217, 12218, 12219, 12220, 12221, 12222, 12223, 12224, 12225,
+ 12226, 12227, 12228, 12229, 12230, 12231, 12232, 12233, 12234, 12235, 12236, 12237, 12238,
+ 12239, 12240, 12241, 12242, 12243, 12244, 12245, 12288, 12342, 12344, 12345, 12346, 12443,
+ 12444, 12447, 12543, 12593, 12594, 12595, 12596, 12597, 12598, 12599, 12600, 12601, 12602,
+ 12603, 12604, 12605, 12606, 12607, 12608, 12609, 12610, 12611, 12612, 12613, 12614, 12615,
+ 12616, 12617, 12618, 12619, 12620, 12621, 12622, 12623, 12624, 12625, 12626, 12627, 12628,
+ 12629, 12630, 12631, 12632, 12633, 12634, 12635, 12636, 12637, 12638, 12639, 12640, 12641,
+ 12642, 12643, 12644, 12645, 12646, 12647, 12648, 12649, 12650, 12651, 12652, 12653, 12654,
+ 12655, 12656, 12657, 12658, 12659, 12660, 12661, 12662, 12663, 12664, 12665, 12666, 12667,
+ 12668, 12669, 12670, 12671, 12672, 12673, 12674, 12675, 12676, 12677, 12678, 12679, 12680,
+ 12681, 12682, 12683, 12684, 12685, 12686, 12690, 12691, 12692, 12693, 12694, 12695, 12696,
+ 12697, 12698, 12699, 12700, 12701, 12702, 12703, 12800, 12801, 12802, 12803, 12804, 12805,
+ 12806, 12807, 12808, 12809, 12810, 12811, 12812, 12813, 12814, 12815, 12816, 12817, 12818,
+ 12819, 12820, 12821, 12822, 12823, 12824, 12825, 12826, 12827, 12828, 12829, 12830, 12832,
+ 12833, 12834, 12835, 12836, 12837, 12838, 12839, 12840, 12841, 12842, 12843, 12844, 12845,
+ 12846, 12847, 12848, 12849, 12850, 12851, 12852, 12853, 12854, 12855, 12856, 12857, 12858,
+ 12859, 12860, 12861, 12862, 12863, 12864, 12865, 12866, 12867, 12868, 12869, 12870, 12871,
+ 12880, 12881, 12882, 12883, 12884, 12885, 12886, 12887, 12888, 12889, 12890, 12891, 12892,
+ 12893, 12894, 12895, 12896, 12897, 12898, 12899, 12900, 12901, 12902, 12903, 12904, 12905,
+ 12906, 12907, 12908, 12909, 12910, 12911, 12912, 12913, 12914, 12915, 12916, 12917, 12918,
+ 12919, 12920, 12921, 12922, 12923, 12924, 12925, 12926, 12928, 12929, 12930, 12931, 12932,
+ 12933, 12934, 12935, 12936, 12937, 12938, 12939, 12940, 12941, 12942, 12943, 12944, 12945,
+ 12946, 12947, 12948, 12949, 12950, 12951, 12952, 12953, 12954, 12955, 12956, 12957, 12958,
+ 12959, 12960, 12961, 12962, 12963, 12964, 12965, 12966, 12967, 12968, 12969, 12970, 12971,
+ 12972, 12973, 12974, 12975, 12976, 12977, 12978, 12979, 12980, 12981, 12982, 12983, 12984,
+ 12985, 12986, 12987, 12988, 12989, 12990, 12991, 12992, 12993, 12994, 12995, 12996, 12997,
+ 12998, 12999, 13000, 13001, 13002, 13003, 13004, 13005, 13006, 13007, 13008, 13009, 13010,
+ 13011, 13012, 13013, 13014, 13015, 13016, 13017, 13018, 13019, 13020, 13021, 13022, 13023,
+ 13024, 13025, 13026, 13027, 13028, 13029, 13030, 13031, 13032, 13033, 13034, 13035, 13036,
+ 13037, 13038, 13039, 13040, 13041, 13042, 13043, 13044, 13045, 13046, 13047, 13048, 13049,
+ 13050, 13051, 13052, 13053, 13054, 13056, 13057, 13058, 13059, 13060, 13061, 13062, 13063,
+ 13064, 13065, 13066, 13067, 13068, 13069, 13070, 13071, 13072, 13073, 13074, 13075, 13076,
+ 13077, 13078, 13079, 13080, 13081, 13082, 13083, 13084, 13085, 13086, 13087, 13088, 13089,
+ 13090, 13091, 13092, 13093, 13094, 13095, 13096, 13097, 13098, 13099, 13100, 13101, 13102,
+ 13103, 13104, 13105, 13106, 13107, 13108, 13109, 13110, 13111, 13112, 13113, 13114, 13115,
+ 13116, 13117, 13118, 13119, 13120, 13121, 13122, 13123, 13124, 13125, 13126, 13127, 13128,
+ 13129, 13130, 13131, 13132, 13133, 13134, 13135, 13136, 13137, 13138, 13139, 13140, 13141,
+ 13142, 13143, 13144, 13145, 13146, 13147, 13148, 13149, 13150, 13151, 13152, 13153, 13154,
+ 13155, 13156, 13157, 13158, 13159, 13160, 13161, 13162, 13163, 13164, 13165, 13166, 13167,
+ 13168, 13169, 13170, 13171, 13172, 13173, 13174, 13175, 13176, 13177, 13178, 13179, 13180,
+ 13181, 13182, 13183, 13184, 13185, 13186, 13187, 13188, 13189, 13190, 13191, 13192, 13193,
+ 13194, 13195, 13196, 13197, 13198, 13199, 13200, 13201, 13202, 13203, 13204, 13205, 13206,
+ 13207, 13208, 13209, 13210, 13211, 13212, 13213, 13214, 13215, 13216, 13217, 13218, 13219,
+ 13220, 13221, 13222, 13223, 13224, 13225, 13226, 13227, 13228, 13229, 13230, 13231, 13232,
+ 13233, 13234, 13235, 13236, 13237, 13238, 13239, 13240, 13241, 13242, 13243, 13244, 13245,
+ 13246, 13247, 13248, 13249, 13250, 13251, 13252, 13253, 13254, 13255, 13256, 13257, 13258,
+ 13259, 13260, 13261, 13262, 13263, 13264, 13265, 13266, 13267, 13268, 13269, 13270, 13271,
+ 13272, 13273, 13274, 13275, 13276, 13277, 13278, 13279, 13280, 13281, 13282, 13283, 13284,
+ 13285, 13286, 13287, 13288, 13289, 13290, 13291, 13292, 13293, 13294, 13295, 13296, 13297,
+ 13298, 13299, 13300, 13301, 13302, 13303, 13304, 13305, 13306, 13307, 13308, 13309, 13310,
+ 13311, 42652, 42653, 42864, 43000, 43001, 43868, 43869, 43870, 43871, 64256, 64257, 64258,
+ 64259, 64260, 64261, 64262, 64275, 64276, 64277, 64278, 64279, 64288, 64289, 64290, 64291,
+ 64292, 64293, 64294, 64295, 64296, 64297, 64335, 64336, 64337, 64338, 64339, 64340, 64341,
+ 64342, 64343, 64344, 64345, 64346, 64347, 64348, 64349, 64350, 64351, 64352, 64353, 64354,
+ 64355, 64356, 64357, 64358, 64359, 64360, 64361, 64362, 64363, 64364, 64365, 64366, 64367,
+ 64368, 64369, 64370, 64371, 64372, 64373, 64374, 64375, 64376, 64377, 64378, 64379, 64380,
+ 64381, 64382, 64383, 64384, 64385, 64386, 64387, 64388, 64389, 64390, 64391, 64392, 64393,
+ 64394, 64395, 64396, 64397, 64398, 64399, 64400, 64401, 64402, 64403, 64404, 64405, 64406,
+ 64407, 64408, 64409, 64410, 64411, 64412, 64413, 64414, 64415, 64416, 64417, 64418, 64419,
+ 64420, 64421, 64422, 64423, 64424, 64425, 64426, 64427, 64428, 64429, 64430, 64431, 64432,
+ 64433, 64467, 64468, 64469, 64470, 64471, 64472, 64473, 64474, 64475, 64476, 64477, 64478,
+ 64479, 64480, 64481, 64482, 64483, 64484, 64485, 64486, 64487, 64488, 64489, 64490, 64491,
+ 64492, 64493, 64494, 64495, 64496, 64497, 64498, 64499, 64500, 64501, 64502, 64503, 64504,
+ 64505, 64506, 64507, 64508, 64509, 64510, 64511, 64512, 64513, 64514, 64515, 64516, 64517,
+ 64518, 64519, 64520, 64521, 64522, 64523, 64524, 64525, 64526, 64527, 64528, 64529, 64530,
+ 64531, 64532, 64533, 64534, 64535, 64536, 64537, 64538, 64539, 64540, 64541, 64542, 64543,
+ 64544, 64545, 64546, 64547, 64548, 64549, 64550, 64551, 64552, 64553, 64554, 64555, 64556,
+ 64557, 64558, 64559, 64560, 64561, 64562, 64563, 64564, 64565, 64566, 64567, 64568, 64569,
+ 64570, 64571, 64572, 64573, 64574, 64575, 64576, 64577, 64578, 64579, 64580, 64581, 64582,
+ 64583, 64584, 64585, 64586, 64587, 64588, 64589, 64590, 64591, 64592, 64593, 64594, 64595,
+ 64596, 64597, 64598, 64599, 64600, 64601, 64602, 64603, 64604, 64605, 64606, 64607, 64608,
+ 64609, 64610, 64611, 64612, 64613, 64614, 64615, 64616, 64617, 64618, 64619, 64620, 64621,
+ 64622, 64623, 64624, 64625, 64626, 64627, 64628, 64629, 64630, 64631, 64632, 64633, 64634,
+ 64635, 64636, 64637, 64638, 64639, 64640, 64641, 64642, 64643, 64644, 64645, 64646, 64647,
+ 64648, 64649, 64650, 64651, 64652, 64653, 64654, 64655, 64656, 64657, 64658, 64659, 64660,
+ 64661, 64662, 64663, 64664, 64665, 64666, 64667, 64668, 64669, 64670, 64671, 64672, 64673,
+ 64674, 64675, 64676, 64677, 64678, 64679, 64680, 64681, 64682, 64683, 64684, 64685, 64686,
+ 64687, 64688, 64689, 64690, 64691, 64692, 64693, 64694, 64695, 64696, 64697, 64698, 64699,
+ 64700, 64701, 64702, 64703, 64704, 64705, 64706, 64707, 64708, 64709, 64710, 64711, 64712,
+ 64713, 64714, 64715, 64716, 64717, 64718, 64719, 64720, 64721, 64722, 64723, 64724, 64725,
+ 64726, 64727, 64728, 64729, 64730, 64731, 64732, 64733, 64734, 64735, 64736, 64737, 64738,
+ 64739, 64740, 64741, 64742, 64743, 64744, 64745, 64746, 64747, 64748, 64749, 64750, 64751,
+ 64752, 64753, 64754, 64755, 64756, 64757, 64758, 64759, 64760, 64761, 64762, 64763, 64764,
+ 64765, 64766, 64767, 64768, 64769, 64770, 64771, 64772, 64773, 64774, 64775, 64776, 64777,
+ 64778, 64779, 64780, 64781, 64782, 64783, 64784, 64785, 64786, 64787, 64788, 64789, 64790,
+ 64791, 64792, 64793, 64794, 64795, 64796, 64797, 64798, 64799, 64800, 64801, 64802, 64803,
+ 64804, 64805, 64806, 64807, 64808, 64809, 64810, 64811, 64812, 64813, 64814, 64815, 64816,
+ 64817, 64818, 64819, 64820, 64821, 64822, 64823, 64824, 64825, 64826, 64827, 64828, 64829,
+ 64848, 64849, 64850, 64851, 64852, 64853, 64854, 64855, 64856, 64857, 64858, 64859, 64860,
+ 64861, 64862, 64863, 64864, 64865, 64866, 64867, 64868, 64869, 64870, 64871, 64872, 64873,
+ 64874, 64875, 64876, 64877, 64878, 64879, 64880, 64881, 64882, 64883, 64884, 64885, 64886,
+ 64887, 64888, 64889, 64890, 64891, 64892, 64893, 64894, 64895, 64896, 64897, 64898, 64899,
+ 64900, 64901, 64902, 64903, 64904, 64905, 64906, 64907, 64908, 64909, 64910, 64911, 64914,
+ 64915, 64916, 64917, 64918, 64919, 64920, 64921, 64922, 64923, 64924, 64925, 64926, 64927,
+ 64928, 64929, 64930, 64931, 64932, 64933, 64934, 64935, 64936, 64937, 64938, 64939, 64940,
+ 64941, 64942, 64943, 64944, 64945, 64946, 64947, 64948, 64949, 64950, 64951, 64952, 64953,
+ 64954, 64955, 64956, 64957, 64958, 64959, 64960, 64961, 64962, 64963, 64964, 64965, 64966,
+ 64967, 65008, 65009, 65010, 65011, 65012, 65013, 65014, 65015, 65016, 65017, 65018, 65019,
+ 65020, 65040, 65041, 65042, 65043, 65044, 65045, 65046, 65047, 65048, 65049, 65072, 65073,
+ 65074, 65075, 65076, 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086,
+ 65087, 65088, 65089, 65090, 65091, 65092, 65095, 65096, 65097, 65098, 65099, 65100, 65101,
+ 65102, 65103, 65104, 65105, 65106, 65108, 65109, 65110, 65111, 65112, 65113, 65114, 65115,
+ 65116, 65117, 65118, 65119, 65120, 65121, 65122, 65123, 65124, 65125, 65126, 65128, 65129,
+ 65130, 65131, 65136, 65137, 65138, 65140, 65142, 65143, 65144, 65145, 65146, 65147, 65148,
+ 65149, 65150, 65151, 65152, 65153, 65154, 65155, 65156, 65157, 65158, 65159, 65160, 65161,
+ 65162, 65163, 65164, 65165, 65166, 65167, 65168, 65169, 65170, 65171, 65172, 65173, 65174,
+ 65175, 65176, 65177, 65178, 65179, 65180, 65181, 65182, 65183, 65184, 65185, 65186, 65187,
+ 65188, 65189, 65190, 65191, 65192, 65193, 65194, 65195, 65196, 65197, 65198, 65199, 65200,
+ 65201, 65202, 65203, 65204, 65205, 65206, 65207, 65208, 65209, 65210, 65211, 65212, 65213,
+ 65214, 65215, 65216, 65217, 65218, 65219, 65220, 65221, 65222, 65223, 65224, 65225, 65226,
+ 65227, 65228, 65229, 65230, 65231, 65232, 65233, 65234, 65235, 65236, 65237, 65238, 65239,
+ 65240, 65241, 65242, 65243, 65244, 65245, 65246, 65247, 65248, 65249, 65250, 65251, 65252,
+ 65253, 65254, 65255, 65256, 65257, 65258, 65259, 65260, 65261, 65262, 65263, 65264, 65265,
+ 65266, 65267, 65268, 65269, 65270, 65271, 65272, 65273, 65274, 65275, 65276, 65281, 65282,
+ 65283, 65284, 65285, 65286, 65287, 65288, 65289, 65290, 65291, 65292, 65293, 65294, 65295,
+ 65296, 65297, 65298, 65299, 65300, 65301, 65302, 65303, 65304, 65305, 65306, 65307, 65308,
+ 65309, 65310, 65311, 65312, 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321,
+ 65322, 65323, 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, 65334,
+ 65335, 65336, 65337, 65338, 65339, 65340, 65341, 65342, 65343, 65344, 65345, 65346, 65347,
+ 65348, 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, 65360,
+ 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, 65369, 65370, 65371, 65372, 65373,
+ 65374, 65375, 65376, 65377, 65378, 65379, 65380, 65381, 65382, 65383, 65384, 65385, 65386,
+ 65387, 65388, 65389, 65390, 65391, 65392, 65393, 65394, 65395, 65396, 65397, 65398, 65399,
+ 65400, 65401, 65402, 65403, 65404, 65405, 65406, 65407, 65408, 65409, 65410, 65411, 65412,
+ 65413, 65414, 65415, 65416, 65417, 65418, 65419, 65420, 65421, 65422, 65423, 65424, 65425,
+ 65426, 65427, 65428, 65429, 65430, 65431, 65432, 65433, 65434, 65435, 65436, 65437, 65438,
+ 65439, 65440, 65441, 65442, 65443, 65444, 65445, 65446, 65447, 65448, 65449, 65450, 65451,
+ 65452, 65453, 65454, 65455, 65456, 65457, 65458, 65459, 65460, 65461, 65462, 65463, 65464,
+ 65465, 65466, 65467, 65468, 65469, 65470, 65474, 65475, 65476, 65477, 65478, 65479, 65482,
+ 65483, 65484, 65485, 65486, 65487, 65490, 65491, 65492, 65493, 65494, 65495, 65498, 65499,
+ 65500, 65504, 65505, 65506, 65507, 65508, 65509, 65510, 65512, 65513, 65514, 65515, 65516,
+ 65517, 65518, 119808, 119809, 119810, 119811, 119812, 119813, 119814, 119815, 119816, 119817,
+ 119818, 119819, 119820, 119821, 119822, 119823, 119824, 119825, 119826, 119827, 119828, 119829,
+ 119830, 119831, 119832, 119833, 119834, 119835, 119836, 119837, 119838, 119839, 119840, 119841,
+ 119842, 119843, 119844, 119845, 119846, 119847, 119848, 119849, 119850, 119851, 119852, 119853,
+ 119854, 119855, 119856, 119857, 119858, 119859, 119860, 119861, 119862, 119863, 119864, 119865,
+ 119866, 119867, 119868, 119869, 119870, 119871, 119872, 119873, 119874, 119875, 119876, 119877,
+ 119878, 119879, 119880, 119881, 119882, 119883, 119884, 119885, 119886, 119887, 119888, 119889,
+ 119890, 119891, 119892, 119894, 119895, 119896, 119897, 119898, 119899, 119900, 119901, 119902,
+ 119903, 119904, 119905, 119906, 119907, 119908, 119909, 119910, 119911, 119912, 119913, 119914,
+ 119915, 119916, 119917, 119918, 119919, 119920, 119921, 119922, 119923, 119924, 119925, 119926,
+ 119927, 119928, 119929, 119930, 119931, 119932, 119933, 119934, 119935, 119936, 119937, 119938,
+ 119939, 119940, 119941, 119942, 119943, 119944, 119945, 119946, 119947, 119948, 119949, 119950,
+ 119951, 119952, 119953, 119954, 119955, 119956, 119957, 119958, 119959, 119960, 119961, 119962,
+ 119963, 119964, 119966, 119967, 119970, 119973, 119974, 119977, 119978, 119979, 119980, 119982,
+ 119983, 119984, 119985, 119986, 119987, 119988, 119989, 119990, 119991, 119992, 119993, 119995,
+ 119997, 119998, 119999, 120000, 120001, 120002, 120003, 120005, 120006, 120007, 120008, 120009,
+ 120010, 120011, 120012, 120013, 120014, 120015, 120016, 120017, 120018, 120019, 120020, 120021,
+ 120022, 120023, 120024, 120025, 120026, 120027, 120028, 120029, 120030, 120031, 120032, 120033,
+ 120034, 120035, 120036, 120037, 120038, 120039, 120040, 120041, 120042, 120043, 120044, 120045,
+ 120046, 120047, 120048, 120049, 120050, 120051, 120052, 120053, 120054, 120055, 120056, 120057,
+ 120058, 120059, 120060, 120061, 120062, 120063, 120064, 120065, 120066, 120067, 120068, 120069,
+ 120071, 120072, 120073, 120074, 120077, 120078, 120079, 120080, 120081, 120082, 120083, 120084,
+ 120086, 120087, 120088, 120089, 120090, 120091, 120092, 120094, 120095, 120096, 120097, 120098,
+ 120099, 120100, 120101, 120102, 120103, 120104, 120105, 120106, 120107, 120108, 120109, 120110,
+ 120111, 120112, 120113, 120114, 120115, 120116, 120117, 120118, 120119, 120120, 120121, 120123,
+ 120124, 120125, 120126, 120128, 120129, 120130, 120131, 120132, 120134, 120138, 120139, 120140,
+ 120141, 120142, 120143, 120144, 120146, 120147, 120148, 120149, 120150, 120151, 120152, 120153,
+ 120154, 120155, 120156, 120157, 120158, 120159, 120160, 120161, 120162, 120163, 120164, 120165,
+ 120166, 120167, 120168, 120169, 120170, 120171, 120172, 120173, 120174, 120175, 120176, 120177,
+ 120178, 120179, 120180, 120181, 120182, 120183, 120184, 120185, 120186, 120187, 120188, 120189,
+ 120190, 120191, 120192, 120193, 120194, 120195, 120196, 120197, 120198, 120199, 120200, 120201,
+ 120202, 120203, 120204, 120205, 120206, 120207, 120208, 120209, 120210, 120211, 120212, 120213,
+ 120214, 120215, 120216, 120217, 120218, 120219, 120220, 120221, 120222, 120223, 120224, 120225,
+ 120226, 120227, 120228, 120229, 120230, 120231, 120232, 120233, 120234, 120235, 120236, 120237,
+ 120238, 120239, 120240, 120241, 120242, 120243, 120244, 120245, 120246, 120247, 120248, 120249,
+ 120250, 120251, 120252, 120253, 120254, 120255, 120256, 120257, 120258, 120259, 120260, 120261,
+ 120262, 120263, 120264, 120265, 120266, 120267, 120268, 120269, 120270, 120271, 120272, 120273,
+ 120274, 120275, 120276, 120277, 120278, 120279, 120280, 120281, 120282, 120283, 120284, 120285,
+ 120286, 120287, 120288, 120289, 120290, 120291, 120292, 120293, 120294, 120295, 120296, 120297,
+ 120298, 120299, 120300, 120301, 120302, 120303, 120304, 120305, 120306, 120307, 120308, 120309,
+ 120310, 120311, 120312, 120313, 120314, 120315, 120316, 120317, 120318, 120319, 120320, 120321,
+ 120322, 120323, 120324, 120325, 120326, 120327, 120328, 120329, 120330, 120331, 120332, 120333,
+ 120334, 120335, 120336, 120337, 120338, 120339, 120340, 120341, 120342, 120343, 120344, 120345,
+ 120346, 120347, 120348, 120349, 120350, 120351, 120352, 120353, 120354, 120355, 120356, 120357,
+ 120358, 120359, 120360, 120361, 120362, 120363, 120364, 120365, 120366, 120367, 120368, 120369,
+ 120370, 120371, 120372, 120373, 120374, 120375, 120376, 120377, 120378, 120379, 120380, 120381,
+ 120382, 120383, 120384, 120385, 120386, 120387, 120388, 120389, 120390, 120391, 120392, 120393,
+ 120394, 120395, 120396, 120397, 120398, 120399, 120400, 120401, 120402, 120403, 120404, 120405,
+ 120406, 120407, 120408, 120409, 120410, 120411, 120412, 120413, 120414, 120415, 120416, 120417,
+ 120418, 120419, 120420, 120421, 120422, 120423, 120424, 120425, 120426, 120427, 120428, 120429,
+ 120430, 120431, 120432, 120433, 120434, 120435, 120436, 120437, 120438, 120439, 120440, 120441,
+ 120442, 120443, 120444, 120445, 120446, 120447, 120448, 120449, 120450, 120451, 120452, 120453,
+ 120454, 120455, 120456, 120457, 120458, 120459, 120460, 120461, 120462, 120463, 120464, 120465,
+ 120466, 120467, 120468, 120469, 120470, 120471, 120472, 120473, 120474, 120475, 120476, 120477,
+ 120478, 120479, 120480, 120481, 120482, 120483, 120484, 120485, 120488, 120489, 120490, 120491,
+ 120492, 120493, 120494, 120495, 120496, 120497, 120498, 120499, 120500, 120501, 120502, 120503,
+ 120504, 120505, 120506, 120507, 120508, 120509, 120510, 120511, 120512, 120513, 120514, 120515,
+ 120516, 120517, 120518, 120519, 120520, 120521, 120522, 120523, 120524, 120525, 120526, 120527,
+ 120528, 120529, 120530, 120531, 120532, 120533, 120534, 120535, 120536, 120537, 120538, 120539,
+ 120540, 120541, 120542, 120543, 120544, 120545, 120546, 120547, 120548, 120549, 120550, 120551,
+ 120552, 120553, 120554, 120555, 120556, 120557, 120558, 120559, 120560, 120561, 120562, 120563,
+ 120564, 120565, 120566, 120567, 120568, 120569, 120570, 120571, 120572, 120573, 120574, 120575,
+ 120576, 120577, 120578, 120579, 120580, 120581, 120582, 120583, 120584, 120585, 120586, 120587,
+ 120588, 120589, 120590, 120591, 120592, 120593, 120594, 120595, 120596, 120597, 120598, 120599,
+ 120600, 120601, 120602, 120603, 120604, 120605, 120606, 120607, 120608, 120609, 120610, 120611,
+ 120612, 120613, 120614, 120615, 120616, 120617, 120618, 120619, 120620, 120621, 120622, 120623,
+ 120624, 120625, 120626, 120627, 120628, 120629, 120630, 120631, 120632, 120633, 120634, 120635,
+ 120636, 120637, 120638, 120639, 120640, 120641, 120642, 120643, 120644, 120645, 120646, 120647,
+ 120648, 120649, 120650, 120651, 120652, 120653, 120654, 120655, 120656, 120657, 120658, 120659,
+ 120660, 120661, 120662, 120663, 120664, 120665, 120666, 120667, 120668, 120669, 120670, 120671,
+ 120672, 120673, 120674, 120675, 120676, 120677, 120678, 120679, 120680, 120681, 120682, 120683,
+ 120684, 120685, 120686, 120687, 120688, 120689, 120690, 120691, 120692, 120693, 120694, 120695,
+ 120696, 120697, 120698, 120699, 120700, 120701, 120702, 120703, 120704, 120705, 120706, 120707,
+ 120708, 120709, 120710, 120711, 120712, 120713, 120714, 120715, 120716, 120717, 120718, 120719,
+ 120720, 120721, 120722, 120723, 120724, 120725, 120726, 120727, 120728, 120729, 120730, 120731,
+ 120732, 120733, 120734, 120735, 120736, 120737, 120738, 120739, 120740, 120741, 120742, 120743,
+ 120744, 120745, 120746, 120747, 120748, 120749, 120750, 120751, 120752, 120753, 120754, 120755,
+ 120756, 120757, 120758, 120759, 120760, 120761, 120762, 120763, 120764, 120765, 120766, 120767,
+ 120768, 120769, 120770, 120771, 120772, 120773, 120774, 120775, 120776, 120777, 120778, 120779,
+ 120782, 120783, 120784, 120785, 120786, 120787, 120788, 120789, 120790, 120791, 120792, 120793,
+ 120794, 120795, 120796, 120797, 120798, 120799, 120800, 120801, 120802, 120803, 120804, 120805,
+ 120806, 120807, 120808, 120809, 120810, 120811, 120812, 120813, 120814, 120815, 120816, 120817,
+ 120818, 120819, 120820, 120821, 120822, 120823, 120824, 120825, 120826, 120827, 120828, 120829,
+ 120830, 120831, 126464, 126465, 126466, 126467, 126469, 126470, 126471, 126472, 126473, 126474,
+ 126475, 126476, 126477, 126478, 126479, 126480, 126481, 126482, 126483, 126484, 126485, 126486,
+ 126487, 126488, 126489, 126490, 126491, 126492, 126493, 126494, 126495, 126497, 126498, 126500,
+ 126503, 126505, 126506, 126507, 126508, 126509, 126510, 126511, 126512, 126513, 126514, 126516,
+ 126517, 126518, 126519, 126521, 126523, 126530, 126535, 126537, 126539, 126541, 126542, 126543,
+ 126545, 126546, 126548, 126551, 126553, 126555, 126557, 126559, 126561, 126562, 126564, 126567,
+ 126568, 126569, 126570, 126572, 126573, 126574, 126575, 126576, 126577, 126578, 126580, 126581,
+ 126582, 126583, 126585, 126586, 126587, 126588, 126590, 126592, 126593, 126594, 126595, 126596,
+ 126597, 126598, 126599, 126600, 126601, 126603, 126604, 126605, 126606, 126607, 126608, 126609,
+ 126610, 126611, 126612, 126613, 126614, 126615, 126616, 126617, 126618, 126619, 126625, 126626,
+ 126627, 126629, 126630, 126631, 126632, 126633, 126635, 126636, 126637, 126638, 126639, 126640,
+ 126641, 126642, 126643, 126644, 126645, 126646, 126647, 126648, 126649, 126650, 126651, 127232,
+ 127233, 127234, 127235, 127236, 127237, 127238, 127239, 127240, 127241, 127242, 127248, 127249,
+ 127250, 127251, 127252, 127253, 127254, 127255, 127256, 127257, 127258, 127259, 127260, 127261,
+ 127262, 127263, 127264, 127265, 127266, 127267, 127268, 127269, 127270, 127271, 127272, 127273,
+ 127274, 127275, 127276, 127277, 127278, 127280, 127281, 127282, 127283, 127284, 127285, 127286,
+ 127287, 127288, 127289, 127290, 127291, 127292, 127293, 127294, 127295, 127296, 127297, 127298,
+ 127299, 127300, 127301, 127302, 127303, 127304, 127305, 127306, 127307, 127308, 127309, 127310,
+ 127311, 127338, 127339, 127376, 127488, 127489, 127490, 127504, 127505, 127506, 127507, 127508,
+ 127509, 127510, 127511, 127512, 127513, 127514, 127515, 127516, 127517, 127518, 127519, 127520,
+ 127521, 127522, 127523, 127524, 127525, 127526, 127527, 127528, 127529, 127530, 127531, 127532,
+ 127533, 127534, 127535, 127536, 127537, 127538, 127539, 127540, 127541, 127542, 127543, 127544,
+ 127545, 127546, 127547, 127552, 127553, 127554, 127555, 127556, 127557, 127558, 127559, 127560,
+ 127568, 127569,
+};
+
+// offsets into k_compat_pool
+inline constexpr uint32_t k_compat_offsets[] = {
+ 0, 1, 3, 4, 6, 7, 8, 10, 11, 13, 14, 15, 18, 21, 24, 26, 28, 30, 32, 34, 35, 38, 41, 44, 46,
+ 48, 50, 52, 54, 56, 58, 60, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 73, 75, 77, 79, 81, 83, 84,
+ 85, 86, 87, 88, 90, 92, 95, 96, 97, 98, 100, 102, 103, 104, 105, 106, 107, 108, 109, 110, 112,
+ 114, 116, 118, 120, 122, 124, 126, 128, 129, 132, 135, 136, 137, 138, 139, 140, 141, 142, 143,
+ 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162,
+ 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181,
+ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200,
+ 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,
+ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 236, 238, 240, 242,
+ 244, 247, 250, 253, 256, 259, 262, 265, 268, 271, 273, 275, 276, 277, 278, 279, 280, 281, 282,
+ 283, 284, 285, 286, 287, 289, 290, 292, 295, 296, 298, 301, 303, 306, 308, 310, 312, 314, 316,
+ 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338,
+ 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357,
+ 358, 359, 360, 361, 362, 363, 365, 368, 371, 372, 374, 377, 380, 381, 383, 384, 385, 386, 387,
+ 388, 389, 390, 391, 392, 393, 394, 396, 397, 398, 399, 400, 401, 403, 406, 408, 409, 410, 411,
+ 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 425, 426, 427, 428, 429, 430, 431, 432,
+ 433, 434, 435, 438, 441, 445, 448, 451, 454, 457, 460, 463, 466, 469, 472, 475, 478, 481, 483,
+ 484, 486, 489, 491, 492, 494, 497, 501, 503, 504, 506, 509, 510, 511, 512, 513, 514, 516, 519,
+ 521, 522, 524, 527, 531, 533, 534, 536, 539, 540, 541, 542, 543, 546, 548, 551, 553, 556, 557,
+ 558, 559, 560, 561, 562, 563, 564, 565, 567, 569, 571, 573, 575, 577, 579, 581, 583, 585, 587,
+ 590, 593, 596, 599, 602, 605, 608, 611, 614, 618, 622, 626, 630, 634, 638, 642, 646, 650, 654,
+ 658, 660, 662, 664, 666, 668, 670, 672, 674, 676, 679, 682, 685, 688, 691, 694, 697, 700, 703,
+ 706, 709, 712, 715, 718, 721, 724, 727, 730, 733, 736, 739, 742, 745, 748, 751, 754, 757, 760,
+ 763, 766, 769, 772, 775, 778, 781, 784, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797,
+ 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816,
+ 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835,
+ 836, 837, 838, 839, 840, 844, 847, 849, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862,
+ 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881,
+ 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900,
+ 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919,
+ 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938,
+ 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957,
+ 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976,
+ 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995,
+ 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011,
+ 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027,
+ 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043,
+ 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059,
+ 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075,
+ 1076, 1078, 1080, 1082, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095,
+ 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111,
+ 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127,
+ 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143,
+ 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159,
+ 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175,
+ 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191,
+ 1192, 1195, 1198, 1201, 1204, 1207, 1210, 1213, 1216, 1219, 1222, 1225, 1228, 1231, 1234, 1238,
+ 1242, 1246, 1250, 1254, 1258, 1262, 1266, 1270, 1274, 1278, 1282, 1286, 1290, 1294, 1301, 1307,
+ 1310, 1313, 1316, 1319, 1322, 1325, 1328, 1331, 1334, 1337, 1340, 1343, 1346, 1349, 1352, 1355,
+ 1358, 1361, 1364, 1367, 1370, 1373, 1376, 1379, 1382, 1385, 1388, 1391, 1394, 1397, 1400, 1403,
+ 1406, 1409, 1412, 1415, 1416, 1417, 1418, 1419, 1422, 1424, 1426, 1428, 1430, 1432, 1434, 1436,
+ 1438, 1440, 1442, 1444, 1446, 1448, 1450, 1452, 1453, 1454, 1455, 1456, 1457, 1458, 1459, 1460,
+ 1461, 1462, 1463, 1464, 1465, 1466, 1468, 1470, 1472, 1474, 1476, 1478, 1480, 1482, 1484, 1486,
+ 1488, 1490, 1492, 1494, 1499, 1503, 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514,
+ 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530,
+ 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546,
+ 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1556, 1558, 1560, 1562, 1564, 1566, 1568, 1570,
+ 1572, 1574, 1576, 1578, 1580, 1582, 1584, 1586, 1588, 1590, 1592, 1594, 1596, 1598, 1600, 1602,
+ 1605, 1608, 1611, 1613, 1616, 1618, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630,
+ 1631, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1646,
+ 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662,
+ 1663, 1664, 1665, 1666, 1667, 1668, 1673, 1677, 1682, 1685, 1690, 1693, 1696, 1702, 1706, 1709,
+ 1712, 1715, 1719, 1723, 1727, 1731, 1735, 1739, 1743, 1749, 1751, 1757, 1763, 1768, 1772, 1778,
+ 1784, 1788, 1791, 1794, 1798, 1802, 1807, 1812, 1815, 1818, 1822, 1825, 1828, 1830, 1832, 1835,
+ 1838, 1844, 1848, 1853, 1859, 1863, 1866, 1869, 1875, 1879, 1885, 1888, 1893, 1896, 1900, 1903,
+ 1907, 1912, 1916, 1921, 1925, 1927, 1932, 1935, 1938, 1942, 1945, 1948, 1951, 1956, 1960, 1962,
+ 1968, 1971, 1976, 1980, 1984, 1987, 1990, 1994, 1996, 2000, 2005, 2007, 2013, 2016, 2018, 2020,
+ 2022, 2024, 2026, 2028, 2030, 2032, 2034, 2036, 2039, 2042, 2045, 2048, 2051, 2054, 2057, 2060,
+ 2063, 2066, 2069, 2072, 2075, 2078, 2081, 2084, 2086, 2088, 2091, 2093, 2095, 2097, 2100, 2103,
+ 2105, 2107, 2109, 2111, 2113, 2117, 2119, 2121, 2123, 2125, 2127, 2129, 2131, 2133, 2136, 2140,
+ 2142, 2144, 2146, 2148, 2150, 2152, 2154, 2157, 2160, 2163, 2166, 2168, 2170, 2172, 2174, 2176,
+ 2178, 2180, 2182, 2184, 2186, 2189, 2192, 2194, 2197, 2200, 2203, 2205, 2208, 2211, 2215, 2217,
+ 2220, 2223, 2226, 2229, 2234, 2240, 2242, 2244, 2246, 2248, 2250, 2252, 2254, 2256, 2258, 2260,
+ 2262, 2264, 2266, 2268, 2270, 2272, 2274, 2276, 2280, 2282, 2284, 2286, 2290, 2293, 2295, 2297,
+ 2299, 2301, 2303, 2305, 2307, 2309, 2311, 2313, 2316, 2318, 2320, 2323, 2326, 2328, 2332, 2335,
+ 2337, 2339, 2341, 2343, 2346, 2349, 2351, 2353, 2355, 2357, 2359, 2361, 2363, 2365, 2367, 2370,
+ 2373, 2376, 2379, 2382, 2385, 2388, 2391, 2394, 2397, 2400, 2403, 2406, 2409, 2412, 2415, 2418,
+ 2421, 2424, 2427, 2430, 2433, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2447,
+ 2449, 2451, 2454, 2457, 2459, 2461, 2463, 2465, 2467, 2469, 2471, 2472, 2473, 2474, 2475, 2476,
+ 2477, 2478, 2479, 2480, 2481, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493,
+ 2494, 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509,
+ 2510, 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525,
+ 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541,
+ 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557,
+ 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2569, 2571, 2572, 2573, 2574, 2575,
+ 2576, 2577, 2578, 2579, 2580, 2581, 2583, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593,
+ 2594, 2595, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, 2612,
+ 2615, 2618, 2621, 2624, 2627, 2630, 2633, 2636, 2639, 2642, 2645, 2648, 2651, 2654, 2657, 2660,
+ 2663, 2664, 2665, 2666, 2667, 2670, 2673, 2676, 2679, 2682, 2684, 2686, 2688, 2690, 2692, 2694,
+ 2696, 2698, 2700, 2702, 2704, 2706, 2708, 2710, 2712, 2714, 2716, 2718, 2720, 2722, 2724, 2726,
+ 2728, 2730, 2732, 2734, 2736, 2738, 2740, 2742, 2744, 2746, 2748, 2750, 2752, 2754, 2756, 2758,
+ 2760, 2762, 2764, 2766, 2768, 2770, 2772, 2774, 2776, 2778, 2780, 2782, 2784, 2786, 2788, 2790,
+ 2792, 2794, 2796, 2798, 2800, 2802, 2804, 2806, 2808, 2810, 2812, 2814, 2816, 2818, 2820, 2822,
+ 2824, 2826, 2828, 2830, 2832, 2834, 2836, 2838, 2840, 2842, 2844, 2846, 2848, 2850, 2852, 2854,
+ 2856, 2858, 2860, 2863, 2866, 2869, 2872, 2875, 2878, 2881, 2884, 2887, 2890, 2893, 2896, 2898,
+ 2900, 2902, 2904, 2906, 2908, 2910, 2912, 2914, 2916, 2918, 2920, 2922, 2924, 2926, 2928, 2930,
+ 2932, 2934, 2936, 2938, 2940, 2942, 2944, 2946, 2948, 2950, 2952, 2954, 2956, 2958, 2960, 2962,
+ 2964, 2966, 2968, 2970, 2972, 2974, 2976, 2978, 2980, 2982, 2984, 2986, 2989, 2992, 2995, 2998,
+ 3001, 3003, 3005, 3007, 3009, 3011, 3013, 3015, 3017, 3019, 3021, 3023, 3025, 3027, 3029, 3031,
+ 3033, 3035, 3037, 3039, 3041, 3043, 3045, 3047, 3049, 3051, 3053, 3055, 3057, 3059, 3061, 3063,
+ 3065, 3067, 3069, 3071, 3073, 3075, 3077, 3079, 3081, 3083, 3085, 3087, 3089, 3091, 3093, 3095,
+ 3097, 3099, 3101, 3103, 3105, 3107, 3109, 3111, 3113, 3115, 3117, 3119, 3121, 3123, 3125, 3127,
+ 3129, 3131, 3133, 3135, 3138, 3141, 3143, 3145, 3147, 3149, 3151, 3153, 3155, 3157, 3159, 3161,
+ 3163, 3165, 3167, 3169, 3171, 3173, 3175, 3178, 3181, 3184, 3186, 3188, 3190, 3192, 3194, 3196,
+ 3198, 3200, 3202, 3204, 3206, 3208, 3210, 3212, 3214, 3216, 3218, 3220, 3222, 3224, 3226, 3228,
+ 3230, 3232, 3234, 3236, 3238, 3240, 3242, 3244, 3246, 3248, 3250, 3252, 3254, 3256, 3258, 3260,
+ 3262, 3264, 3266, 3268, 3270, 3272, 3274, 3276, 3278, 3280, 3282, 3284, 3286, 3288, 3290, 3292,
+ 3294, 3296, 3298, 3300, 3302, 3304, 3306, 3308, 3310, 3312, 3314, 3316, 3318, 3320, 3322, 3324,
+ 3326, 3328, 3330, 3333, 3336, 3339, 3342, 3345, 3348, 3351, 3354, 3357, 3360, 3363, 3366, 3369,
+ 3372, 3375, 3378, 3381, 3384, 3387, 3390, 3393, 3396, 3399, 3402, 3405, 3408, 3411, 3414, 3417,
+ 3420, 3423, 3426, 3429, 3432, 3435, 3438, 3441, 3444, 3447, 3450, 3453, 3456, 3459, 3462, 3465,
+ 3468, 3471, 3474, 3477, 3480, 3483, 3486, 3489, 3492, 3495, 3498, 3501, 3504, 3507, 3510, 3513,
+ 3516, 3519, 3522, 3525, 3528, 3531, 3534, 3537, 3540, 3543, 3546, 3549, 3552, 3555, 3558, 3561,
+ 3564, 3567, 3570, 3573, 3576, 3579, 3582, 3585, 3588, 3591, 3594, 3597, 3600, 3603, 3606, 3609,
+ 3612, 3615, 3618, 3621, 3624, 3627, 3630, 3633, 3636, 3639, 3642, 3645, 3648, 3651, 3654, 3657,
+ 3660, 3663, 3666, 3669, 3672, 3675, 3678, 3681, 3684, 3687, 3690, 3694, 3698, 3702, 3706, 3710,
+ 3714, 3718, 3721, 3739, 3747, 3751, 3752, 3753, 3754, 3755, 3756, 3757, 3758, 3759, 3760, 3763,
+ 3765, 3766, 3767, 3768, 3769, 3770, 3771, 3772, 3773, 3774, 3775, 3776, 3777, 3778, 3779, 3780,
+ 3781, 3782, 3783, 3784, 3785, 3786, 3787, 3789, 3791, 3793, 3795, 3796, 3797, 3798, 3799, 3800,
+ 3801, 3802, 3803, 3804, 3805, 3806, 3807, 3808, 3809, 3810, 3811, 3812, 3813, 3814, 3815, 3816,
+ 3817, 3818, 3819, 3820, 3821, 3822, 3823, 3824, 3826, 3828, 3830, 3832, 3834, 3836, 3838, 3840,
+ 3842, 3844, 3846, 3848, 3850, 3852, 3853, 3855, 3857, 3859, 3861, 3863, 3865, 3867, 3869, 3871,
+ 3873, 3875, 3877, 3878, 3879, 3880, 3881, 3882, 3883, 3884, 3885, 3886, 3887, 3888, 3889, 3890,
+ 3891, 3892, 3893, 3894, 3895, 3896, 3897, 3898, 3899, 3900, 3901, 3902, 3903, 3904, 3905, 3906,
+ 3907, 3908, 3909, 3910, 3911, 3912, 3913, 3914, 3915, 3916, 3917, 3918, 3919, 3920, 3921, 3922,
+ 3923, 3924, 3925, 3926, 3927, 3928, 3929, 3930, 3931, 3932, 3933, 3934, 3935, 3936, 3937, 3938,
+ 3939, 3940, 3941, 3942, 3943, 3944, 3945, 3946, 3947, 3948, 3949, 3950, 3951, 3952, 3953, 3954,
+ 3955, 3956, 3957, 3958, 3959, 3960, 3961, 3962, 3963, 3964, 3965, 3966, 3967, 3968, 3969, 3970,
+ 3971, 3972, 3973, 3974, 3975, 3976, 3977, 3978, 3979, 3980, 3981, 3984, 3987, 3990, 3993, 3996,
+ 3999, 4001, 4003, 4004, 4005, 4006, 4007, 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016,
+ 4017, 4018, 4019, 4020, 4021, 4022, 4023, 4024, 4025, 4026, 4027, 4028, 4029, 4030, 4031, 4032,
+ 4033, 4034, 4035, 4036, 4037, 4038, 4039, 4040, 4041, 4042, 4043, 4044, 4045, 4046, 4047, 4048,
+ 4049, 4050, 4051, 4052, 4053, 4054, 4055, 4056, 4057, 4058, 4059, 4060, 4061, 4062, 4063, 4064,
+ 4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079, 4080,
+ 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095, 4096,
+ 4097, 4098, 4099, 4100, 4101, 4102, 4103, 4104, 4105, 4106, 4107, 4108, 4109, 4110, 4111, 4112,
+ 4113, 4114, 4115, 4116, 4117, 4118, 4119, 4120, 4121, 4122, 4123, 4124, 4125, 4126, 4127, 4128,
+ 4129, 4130, 4131, 4132, 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4143, 4144,
+ 4145, 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, 4159, 4160,
+ 4161, 4162, 4163, 4164, 4165, 4166, 4167, 4168, 4169, 4170, 4171, 4172, 4173, 4174, 4175, 4176,
+ 4177, 4178, 4179, 4180, 4181, 4182, 4183, 4184, 4185, 4186, 4187, 4188, 4189, 4190, 4191, 4192,
+ 4193, 4194, 4195, 4196, 4197, 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4205, 4206, 4207, 4208,
+ 4209, 4210, 4211, 4212, 4213, 4214, 4215, 4216, 4217, 4219, 4220, 4221, 4222, 4223, 4224, 4225,
+ 4226, 4227, 4228, 4229, 4230, 4231, 4232, 4233, 4234, 4235, 4236, 4237, 4238, 4239, 4240, 4241,
+ 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249, 4250, 4251, 4252, 4253, 4254, 4255, 4256, 4257,
+ 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266, 4267, 4268, 4269, 4270, 4271, 4272, 4273,
+ 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282, 4283, 4284, 4285, 4286, 4287, 4288, 4289,
+ 4290, 4291, 4292, 4293, 4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303, 4304, 4305,
+ 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321,
+ 4322, 4323, 4324, 4325, 4326, 4327, 4328, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337,
+ 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353,
+ 4354, 4355, 4356, 4357, 4358, 4359, 4360, 4361, 4362, 4363, 4364, 4365, 4366, 4367, 4368, 4369,
+ 4370, 4371, 4372, 4373, 4374, 4375, 4376, 4377, 4378, 4379, 4380, 4381, 4382, 4383, 4384, 4385,
+ 4386, 4387, 4388, 4389, 4390, 4391, 4392, 4393, 4394, 4395, 4396, 4397, 4398, 4399, 4400, 4401,
+ 4402, 4403, 4404, 4405, 4406, 4407, 4408, 4409, 4410, 4411, 4412, 4413, 4414, 4415, 4416, 4417,
+ 4418, 4419, 4420, 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, 4429, 4430, 4431, 4432, 4433,
+ 4434, 4435, 4436, 4437, 4438, 4439, 4440, 4441, 4442, 4443, 4444, 4445, 4446, 4447, 4448, 4449,
+ 4450, 4451, 4452, 4453, 4454, 4455, 4456, 4457, 4458, 4459, 4460, 4461, 4462, 4463, 4464, 4465,
+ 4466, 4467, 4468, 4469, 4470, 4471, 4472, 4473, 4474, 4475, 4476, 4477, 4478, 4479, 4480, 4481,
+ 4482, 4483, 4484, 4485, 4486, 4487, 4488, 4489, 4490, 4491, 4492, 4493, 4494, 4495, 4496, 4497,
+ 4498, 4499, 4500, 4501, 4502, 4503, 4504, 4505, 4506, 4507, 4508, 4509, 4510, 4511, 4512, 4513,
+ 4514, 4515, 4516, 4517, 4518, 4519, 4520, 4521, 4522, 4523, 4524, 4525, 4526, 4527, 4528, 4529,
+ 4530, 4531, 4532, 4533, 4534, 4535, 4536, 4537, 4538, 4539, 4540, 4541, 4542, 4543, 4544, 4545,
+ 4546, 4547, 4548, 4549, 4550, 4551, 4552, 4553, 4554, 4555, 4556, 4557, 4558, 4559, 4560, 4561,
+ 4562, 4563, 4564, 4565, 4566, 4567, 4568, 4569, 4570, 4571, 4572, 4573, 4574, 4575, 4576, 4577,
+ 4578, 4579, 4580, 4581, 4582, 4583, 4584, 4585, 4586, 4587, 4588, 4589, 4590, 4591, 4592, 4593,
+ 4594, 4595, 4596, 4597, 4598, 4599, 4600, 4601, 4602, 4603, 4604, 4605, 4606, 4607, 4608, 4609,
+ 4610, 4611, 4612, 4613, 4614, 4615, 4616, 4617, 4618, 4619, 4620, 4621, 4622, 4623, 4624, 4625,
+ 4626, 4627, 4628, 4629, 4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 4640, 4641,
+ 4642, 4643, 4644, 4645, 4646, 4647, 4648, 4649, 4650, 4651, 4652, 4653, 4654, 4655, 4656, 4657,
+ 4658, 4659, 4660, 4661, 4662, 4663, 4664, 4665, 4666, 4667, 4668, 4669, 4670, 4671, 4672, 4673,
+ 4674, 4675, 4676, 4677, 4678, 4679, 4680, 4681, 4682, 4683, 4684, 4685, 4686, 4687, 4688, 4689,
+ 4690, 4691, 4692, 4693, 4694, 4695, 4696, 4697, 4698, 4699, 4700, 4701, 4702, 4703, 4704, 4705,
+ 4706, 4707, 4708, 4709, 4710, 4711, 4712, 4713, 4714, 4715, 4716, 4717, 4718, 4719, 4720, 4721,
+ 4722, 4723, 4724, 4725, 4726, 4727, 4728, 4729, 4730, 4731, 4732, 4733, 4734, 4735, 4736, 4737,
+ 4738, 4739, 4740, 4741, 4742, 4743, 4744, 4745, 4746, 4747, 4748, 4749, 4750, 4751, 4752, 4753,
+ 4754, 4755, 4756, 4757, 4758, 4759, 4760, 4761, 4762, 4763, 4764, 4765, 4766, 4767, 4768, 4769,
+ 4770, 4771, 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, 4780, 4781, 4782, 4783, 4784, 4785,
+ 4786, 4787, 4788, 4789, 4790, 4791, 4792, 4793, 4794, 4795, 4796, 4797, 4798, 4799, 4800, 4801,
+ 4802, 4803, 4804, 4805, 4806, 4807, 4808, 4809, 4810, 4811, 4812, 4813, 4814, 4815, 4816, 4817,
+ 4818, 4819, 4820, 4821, 4822, 4823, 4824, 4825, 4826, 4827, 4828, 4829, 4830, 4831, 4832, 4833,
+ 4834, 4835, 4836, 4837, 4838, 4839, 4840, 4841, 4842, 4843, 4844, 4845, 4846, 4847, 4848, 4849,
+ 4850, 4851, 4852, 4853, 4854, 4855, 4856, 4857, 4858, 4859, 4860, 4861, 4862, 4863, 4864, 4865,
+ 4866, 4867, 4868, 4869, 4870, 4871, 4872, 4873, 4874, 4875, 4876, 4877, 4878, 4879, 4880, 4881,
+ 4882, 4883, 4884, 4885, 4886, 4887, 4888, 4889, 4890, 4891, 4892, 4893, 4894, 4895, 4896, 4897,
+ 4898, 4899, 4900, 4901, 4902, 4903, 4904, 4905, 4906, 4907, 4908, 4909, 4910, 4911, 4912, 4913,
+ 4914, 4915, 4916, 4917, 4918, 4919, 4920, 4921, 4922, 4923, 4924, 4925, 4926, 4927, 4928, 4929,
+ 4930, 4931, 4932, 4933, 4934, 4935, 4936, 4937, 4938, 4939, 4940, 4941, 4942, 4943, 4944, 4945,
+ 4946, 4947, 4948, 4949, 4950, 4951, 4952, 4953, 4954, 4955, 4956, 4957, 4958, 4959, 4960, 4961,
+ 4962, 4963, 4964, 4965, 4966, 4967, 4968, 4969, 4970, 4971, 4972, 4973, 4974, 4975, 4976, 4977,
+ 4978, 4979, 4980, 4981, 4982, 4983, 4984, 4985, 4986, 4987, 4988, 4989, 4990, 4991, 4992, 4993,
+ 4994, 4995, 4996, 4997, 4998, 4999, 5000, 5001, 5002, 5003, 5004, 5005, 5006, 5007, 5008, 5009,
+ 5010, 5011, 5012, 5013, 5014, 5015, 5016, 5017, 5018, 5019, 5020, 5021, 5022, 5023, 5024, 5025,
+ 5026, 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041,
+ 5042, 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5054, 5055, 5056, 5057,
+ 5058, 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073,
+ 5074, 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5089,
+ 5090, 5091, 5092, 5093, 5094, 5095, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105,
+ 5106, 5107, 5108, 5109, 5110, 5111, 5112, 5113, 5114, 5115, 5116, 5117, 5118, 5119, 5120, 5121,
+ 5122, 5123, 5124, 5125, 5126, 5127, 5128, 5129, 5130, 5131, 5132, 5133, 5134, 5135, 5136, 5137,
+ 5138, 5139, 5140, 5141, 5142, 5143, 5144, 5145, 5146, 5147, 5148, 5149, 5150, 5151, 5152, 5153,
+ 5154, 5155, 5156, 5157, 5158, 5159, 5160, 5161, 5162, 5163, 5164, 5165, 5166, 5167, 5168, 5169,
+ 5170, 5171, 5172, 5173, 5174, 5175, 5176, 5177, 5178, 5179, 5180, 5181, 5182, 5183, 5184, 5185,
+ 5186, 5187, 5188, 5189, 5190, 5191, 5192, 5193, 5194, 5195, 5196, 5197, 5198, 5199, 5200, 5201,
+ 5202, 5203, 5204, 5205, 5206, 5207, 5208, 5209, 5210, 5211, 5212, 5213, 5214, 5215, 5216, 5217,
+ 5218, 5219, 5220, 5221, 5222, 5223, 5224, 5225, 5226, 5227, 5228, 5229, 5230, 5231, 5232, 5233,
+ 5234, 5235, 5236, 5237, 5238, 5239, 5240, 5241, 5242, 5243, 5244, 5245, 5246, 5247, 5248, 5249,
+ 5250, 5251, 5252, 5253, 5254, 5255, 5256, 5257, 5258, 5259, 5260, 5261, 5262, 5263, 5264, 5265,
+ 5266, 5267, 5268, 5269, 5270, 5271, 5272, 5273, 5274, 5275, 5276, 5277, 5278, 5279, 5280, 5281,
+ 5282, 5283, 5284, 5285, 5286, 5287, 5288, 5289, 5290, 5291, 5292, 5293, 5294, 5295, 5296, 5297,
+ 5298, 5299, 5300, 5301, 5302, 5303, 5304, 5305, 5306, 5307, 5308, 5309, 5310, 5311, 5312, 5313,
+ 5314, 5315, 5316, 5317, 5318, 5319, 5320, 5321, 5322, 5323, 5324, 5325, 5326, 5327, 5328, 5329,
+ 5330, 5331, 5332, 5333, 5334, 5335, 5336, 5337, 5338, 5339, 5340, 5341, 5342, 5343, 5344, 5345,
+ 5346, 5347, 5348, 5349, 5350, 5351, 5352, 5353, 5354, 5355, 5356, 5357, 5358, 5359, 5360, 5361,
+ 5362, 5363, 5364, 5365, 5366, 5368, 5370, 5372, 5374, 5376, 5378, 5380, 5382, 5384, 5386, 5388,
+ 5391, 5394, 5397, 5400, 5403, 5406, 5409, 5412, 5415, 5418, 5421, 5424, 5427, 5430, 5433, 5436,
+ 5439, 5442, 5445, 5448, 5451, 5454, 5457, 5460, 5463, 5466, 5469, 5470, 5471, 5473, 5475, 5476,
+ 5477, 5478, 5479, 5480, 5481, 5482, 5483, 5484, 5485, 5486, 5487, 5488, 5489, 5490, 5491, 5492,
+ 5493, 5494, 5495, 5496, 5497, 5498, 5499, 5500, 5501, 5503, 5505, 5507, 5509, 5512, 5514, 5516,
+ 5518, 5520, 5522, 5524, 5525, 5526, 5527, 5528, 5530, 5531, 5532, 5533, 5534, 5535, 5536, 5537,
+ 5538, 5539, 5540, 5541, 5542, 5543, 5544, 5545, 5546, 5547, 5548, 5549, 5550, 5551, 5552, 5553,
+ 5554, 5555, 5556, 5557, 5558, 5559, 5560, 5561, 5562, 5563, 5564, 5565, 5566, 5567, 5568, 5569,
+ 5570, 5573, 5576, 5579, 5582, 5585, 5588, 5591, 5594, 5597, 5598, 5599,
+};
+
+// the decompositions of k_compat_keys
+inline constexpr uint32_t k_compat_pool[] = {
+ 32, 32, 776, 97, 32, 772, 50, 51, 32, 769, 956, 32, 807, 49, 111, 49, 8260, 52, 49, 8260, 50,
+ 51, 8260, 52, 73, 74, 105, 106, 76, 183, 108, 183, 700, 110, 115, 68, 90, 780, 68, 122, 780,
+ 100, 122, 780, 76, 74, 76, 106, 108, 106, 78, 74, 78, 106, 110, 106, 68, 90, 68, 122, 100, 122,
+ 104, 614, 106, 114, 633, 635, 641, 119, 121, 32, 774, 32, 775, 32, 778, 32, 808, 32, 771, 32,
+ 779, 611, 108, 115, 120, 661, 32, 837, 32, 769, 32, 776, 769, 946, 952, 933, 933, 769, 933,
+ 776, 966, 960, 954, 961, 962, 920, 949, 931, 1381, 1410, 1575, 1652, 1608, 1652, 1735, 1652,
+ 1610, 1652, 3661, 3634, 3789, 3762, 3755, 3737, 3755, 3745, 3851, 4018, 3953, 3968, 4019, 3953,
+ 3968, 4316, 65, 198, 66, 68, 69, 398, 71, 72, 73, 74, 75, 76, 77, 78, 79, 546, 80, 82, 84, 85,
+ 87, 97, 592, 593, 7426, 98, 100, 101, 601, 603, 604, 103, 107, 109, 331, 111, 596, 7446, 7447,
+ 112, 116, 117, 7453, 623, 118, 7461, 946, 947, 948, 966, 967, 105, 114, 117, 118, 946, 947,
+ 961, 966, 967, 1085, 594, 99, 597, 240, 604, 102, 607, 609, 613, 616, 617, 618, 7547, 669, 621,
+ 7557, 671, 625, 624, 626, 627, 628, 629, 632, 642, 643, 427, 649, 650, 7452, 651, 652, 122,
+ 656, 657, 658, 952, 97, 702, 115, 775, 32, 787, 32, 787, 32, 834, 32, 776, 834, 32, 787, 768,
+ 32, 787, 769, 32, 787, 834, 32, 788, 768, 32, 788, 769, 32, 788, 834, 32, 776, 768, 32, 776,
+ 769, 32, 769, 32, 788, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 8208, 32, 819, 46, 46, 46,
+ 46, 46, 46, 32, 8242, 8242, 8242, 8242, 8242, 8245, 8245, 8245, 8245, 8245, 33, 33, 32, 773,
+ 63, 63, 63, 33, 33, 63, 8242, 8242, 8242, 8242, 32, 48, 105, 52, 53, 54, 55, 56, 57, 43, 8722,
+ 61, 40, 41, 110, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 43, 8722, 61, 40, 41, 97, 101, 111,
+ 120, 601, 104, 107, 108, 109, 110, 112, 115, 116, 82, 115, 97, 47, 99, 97, 47, 115, 67, 176,
+ 67, 99, 47, 111, 99, 47, 117, 400, 176, 70, 103, 72, 72, 72, 104, 295, 73, 73, 76, 108, 78, 78,
+ 111, 80, 81, 82, 82, 82, 83, 77, 84, 69, 76, 84, 77, 90, 90, 66, 67, 101, 69, 70, 77, 111,
+ 1488, 1489, 1490, 1491, 105, 70, 65, 88, 960, 947, 915, 928, 8721, 68, 100, 101, 105, 106, 49,
+ 8260, 55, 49, 8260, 57, 49, 8260, 49, 48, 49, 8260, 51, 50, 8260, 51, 49, 8260, 53, 50, 8260,
+ 53, 51, 8260, 53, 52, 8260, 53, 49, 8260, 54, 53, 8260, 54, 49, 8260, 56, 51, 8260, 56, 53,
+ 8260, 56, 55, 8260, 56, 49, 8260, 73, 73, 73, 73, 73, 73, 73, 86, 86, 86, 73, 86, 73, 73, 86,
+ 73, 73, 73, 73, 88, 88, 88, 73, 88, 73, 73, 76, 67, 68, 77, 105, 105, 105, 105, 105, 105, 105,
+ 118, 118, 118, 105, 118, 105, 105, 118, 105, 105, 105, 105, 120, 120, 120, 105, 120, 105, 105,
+ 108, 99, 100, 109, 48, 8260, 51, 8747, 8747, 8747, 8747, 8747, 8750, 8750, 8750, 8750, 8750,
+ 49, 50, 51, 52, 53, 54, 55, 56, 57, 49, 48, 49, 49, 49, 50, 49, 51, 49, 52, 49, 53, 49, 54, 49,
+ 55, 49, 56, 49, 57, 50, 48, 40, 49, 41, 40, 50, 41, 40, 51, 41, 40, 52, 41, 40, 53, 41, 40, 54,
+ 41, 40, 55, 41, 40, 56, 41, 40, 57, 41, 40, 49, 48, 41, 40, 49, 49, 41, 40, 49, 50, 41, 40, 49,
+ 51, 41, 40, 49, 52, 41, 40, 49, 53, 41, 40, 49, 54, 41, 40, 49, 55, 41, 40, 49, 56, 41, 40, 49,
+ 57, 41, 40, 50, 48, 41, 49, 46, 50, 46, 51, 46, 52, 46, 53, 46, 54, 46, 55, 46, 56, 46, 57, 46,
+ 49, 48, 46, 49, 49, 46, 49, 50, 46, 49, 51, 46, 49, 52, 46, 49, 53, 46, 49, 54, 46, 49, 55, 46,
+ 49, 56, 46, 49, 57, 46, 50, 48, 46, 40, 97, 41, 40, 98, 41, 40, 99, 41, 40, 100, 41, 40, 101,
+ 41, 40, 102, 41, 40, 103, 41, 40, 104, 41, 40, 105, 41, 40, 106, 41, 40, 107, 41, 40, 108, 41,
+ 40, 109, 41, 40, 110, 41, 40, 111, 41, 40, 112, 41, 40, 113, 41, 40, 114, 41, 40, 115, 41, 40,
+ 116, 41, 40, 117, 41, 40, 118, 41, 40, 119, 41, 40, 120, 41, 40, 121, 41, 40, 122, 41, 65, 66,
+ 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90,
+ 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,
+ 116, 117, 118, 119, 120, 121, 122, 48, 8747, 8747, 8747, 8747, 58, 58, 61, 61, 61, 61, 61, 61,
+ 106, 86, 11617, 27597, 40863, 19968, 20008, 20022, 20031, 20057, 20101, 20108, 20128, 20154,
+ 20799, 20837, 20843, 20866, 20886, 20907, 20960, 20981, 20992, 21147, 21241, 21269, 21274,
+ 21304, 21313, 21340, 21353, 21378, 21430, 21448, 21475, 22231, 22303, 22763, 22786, 22794,
+ 22805, 22823, 22899, 23376, 23424, 23544, 23567, 23586, 23608, 23662, 23665, 24027, 24037,
+ 24049, 24062, 24178, 24186, 24191, 24308, 24318, 24331, 24339, 24400, 24417, 24435, 24515,
+ 25096, 25142, 25163, 25903, 25908, 25991, 26007, 26020, 26041, 26080, 26085, 26352, 26376,
+ 26408, 27424, 27490, 27513, 27571, 27595, 27604, 27611, 27663, 27668, 27700, 28779, 29226,
+ 29238, 29243, 29247, 29255, 29273, 29275, 29356, 29572, 29577, 29916, 29926, 29976, 29983,
+ 29992, 30000, 30091, 30098, 30326, 30333, 30382, 30399, 30446, 30683, 30690, 30707, 31034,
+ 31160, 31166, 31348, 31435, 31481, 31859, 31992, 32566, 32593, 32650, 32701, 32769, 32780,
+ 32786, 32819, 32895, 32905, 33251, 33258, 33267, 33276, 33292, 33307, 33311, 33390, 33394,
+ 33400, 34381, 34411, 34880, 34892, 34915, 35198, 35211, 35282, 35328, 35895, 35910, 35925,
+ 35960, 35997, 36196, 36208, 36275, 36523, 36554, 36763, 36784, 36789, 37009, 37193, 37318,
+ 37324, 37329, 38263, 38272, 38428, 38582, 38585, 38632, 38737, 38750, 38754, 38761, 38859,
+ 38893, 38899, 38913, 39080, 39131, 39135, 39318, 39321, 39340, 39592, 39640, 39647, 39717,
+ 39727, 39730, 39740, 39770, 40165, 40565, 40575, 40613, 40635, 40643, 40653, 40657, 40697,
+ 40701, 40718, 40723, 40736, 40763, 40778, 40786, 40845, 40860, 40864, 32, 12306, 21313, 21316,
+ 21317, 32, 12441, 32, 12442, 12424, 12426, 12467, 12488, 4352, 4353, 4522, 4354, 4524, 4525,
+ 4355, 4356, 4357, 4528, 4529, 4530, 4531, 4532, 4533, 4378, 4358, 4359, 4360, 4385, 4361, 4362,
+ 4363, 4364, 4365, 4366, 4367, 4368, 4369, 4370, 4449, 4450, 4451, 4452, 4453, 4454, 4455, 4456,
+ 4457, 4458, 4459, 4460, 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, 4469, 4448, 4372, 4373,
+ 4551, 4552, 4556, 4558, 4563, 4567, 4569, 4380, 4573, 4575, 4381, 4382, 4384, 4386, 4387, 4391,
+ 4393, 4395, 4396, 4397, 4398, 4399, 4402, 4406, 4416, 4423, 4428, 4593, 4594, 4439, 4440, 4441,
+ 4484, 4485, 4488, 4497, 4498, 4500, 4510, 4513, 19968, 20108, 19977, 22235, 19978, 20013,
+ 19979, 30002, 20057, 19993, 19969, 22825, 22320, 20154, 40, 4352, 41, 40, 4354, 41, 40, 4355,
+ 41, 40, 4357, 41, 40, 4358, 41, 40, 4359, 41, 40, 4361, 41, 40, 4363, 41, 40, 4364, 41, 40,
+ 4366, 41, 40, 4367, 41, 40, 4368, 41, 40, 4369, 41, 40, 4370, 41, 40, 4352, 4449, 41, 40, 4354,
+ 4449, 41, 40, 4355, 4449, 41, 40, 4357, 4449, 41, 40, 4358, 4449, 41, 40, 4359, 4449, 41, 40,
+ 4361, 4449, 41, 40, 4363, 4449, 41, 40, 4364, 4449, 41, 40, 4366, 4449, 41, 40, 4367, 4449, 41,
+ 40, 4368, 4449, 41, 40, 4369, 4449, 41, 40, 4370, 4449, 41, 40, 4364, 4462, 41, 40, 4363, 4457,
+ 4364, 4453, 4523, 41, 40, 4363, 4457, 4370, 4462, 41, 40, 19968, 41, 40, 20108, 41, 40, 19977,
+ 41, 40, 22235, 41, 40, 20116, 41, 40, 20845, 41, 40, 19971, 41, 40, 20843, 41, 40, 20061, 41,
+ 40, 21313, 41, 40, 26376, 41, 40, 28779, 41, 40, 27700, 41, 40, 26408, 41, 40, 37329, 41, 40,
+ 22303, 41, 40, 26085, 41, 40, 26666, 41, 40, 26377, 41, 40, 31038, 41, 40, 21517, 41, 40,
+ 29305, 41, 40, 36001, 41, 40, 31069, 41, 40, 21172, 41, 40, 20195, 41, 40, 21628, 41, 40,
+ 23398, 41, 40, 30435, 41, 40, 20225, 41, 40, 36039, 41, 40, 21332, 41, 40, 31085, 41, 40,
+ 20241, 41, 40, 33258, 41, 40, 33267, 41, 21839, 24188, 25991, 31631, 80, 84, 69, 50, 49, 50,
+ 50, 50, 51, 50, 52, 50, 53, 50, 54, 50, 55, 50, 56, 50, 57, 51, 48, 51, 49, 51, 50, 51, 51, 51,
+ 52, 51, 53, 4352, 4354, 4355, 4357, 4358, 4359, 4361, 4363, 4364, 4366, 4367, 4368, 4369, 4370,
+ 4352, 4449, 4354, 4449, 4355, 4449, 4357, 4449, 4358, 4449, 4359, 4449, 4361, 4449, 4363, 4449,
+ 4364, 4449, 4366, 4449, 4367, 4449, 4368, 4449, 4369, 4449, 4370, 4449, 4366, 4449, 4535, 4352,
+ 4457, 4364, 4462, 4363, 4468, 4363, 4462, 19968, 20108, 19977, 22235, 20116, 20845, 19971,
+ 20843, 20061, 21313, 26376, 28779, 27700, 26408, 37329, 22303, 26085, 26666, 26377, 31038,
+ 21517, 29305, 36001, 31069, 21172, 31192, 30007, 22899, 36969, 20778, 21360, 27880, 38917,
+ 20241, 20889, 27491, 19978, 20013, 19979, 24038, 21491, 21307, 23447, 23398, 30435, 20225,
+ 36039, 21332, 22812, 51, 54, 51, 55, 51, 56, 51, 57, 52, 48, 52, 49, 52, 50, 52, 51, 52, 52,
+ 52, 53, 52, 54, 52, 55, 52, 56, 52, 57, 53, 48, 49, 26376, 50, 26376, 51, 26376, 52, 26376, 53,
+ 26376, 54, 26376, 55, 26376, 56, 26376, 57, 26376, 49, 48, 26376, 49, 49, 26376, 49, 50, 26376,
+ 72, 103, 101, 114, 103, 101, 86, 76, 84, 68, 12450, 12452, 12454, 12456, 12458, 12459, 12461,
+ 12463, 12465, 12467, 12469, 12471, 12473, 12475, 12477, 12479, 12481, 12484, 12486, 12488,
+ 12490, 12491, 12492, 12493, 12494, 12495, 12498, 12501, 12504, 12507, 12510, 12511, 12512,
+ 12513, 12514, 12516, 12518, 12520, 12521, 12522, 12523, 12524, 12525, 12527, 12528, 12529,
+ 12530, 12450, 12495, 12442, 12540, 12488, 12450, 12523, 12501, 12449, 12450, 12531, 12504,
+ 12442, 12450, 12450, 12540, 12523, 12452, 12491, 12531, 12463, 12441, 12452, 12531, 12481,
+ 12454, 12457, 12531, 12456, 12473, 12463, 12540, 12488, 12441, 12456, 12540, 12459, 12540,
+ 12458, 12531, 12473, 12458, 12540, 12512, 12459, 12452, 12522, 12459, 12521, 12483, 12488,
+ 12459, 12525, 12522, 12540, 12459, 12441, 12525, 12531, 12459, 12441, 12531, 12510, 12461,
+ 12441, 12459, 12441, 12461, 12441, 12491, 12540, 12461, 12517, 12522, 12540, 12461, 12441,
+ 12523, 12479, 12441, 12540, 12461, 12525, 12461, 12525, 12463, 12441, 12521, 12512, 12461,
+ 12525, 12513, 12540, 12488, 12523, 12461, 12525, 12527, 12483, 12488, 12463, 12441, 12521,
+ 12512, 12463, 12441, 12521, 12512, 12488, 12531, 12463, 12523, 12475, 12441, 12452, 12525,
+ 12463, 12525, 12540, 12493, 12465, 12540, 12473, 12467, 12523, 12490, 12467, 12540, 12507,
+ 12442, 12469, 12452, 12463, 12523, 12469, 12531, 12481, 12540, 12512, 12471, 12522, 12531,
+ 12463, 12441, 12475, 12531, 12481, 12475, 12531, 12488, 12479, 12441, 12540, 12473, 12486,
+ 12441, 12471, 12488, 12441, 12523, 12488, 12531, 12490, 12494, 12494, 12483, 12488, 12495,
+ 12452, 12484, 12495, 12442, 12540, 12475, 12531, 12488, 12495, 12442, 12540, 12484, 12495,
+ 12441, 12540, 12524, 12523, 12498, 12442, 12450, 12473, 12488, 12523, 12498, 12442, 12463,
+ 12523, 12498, 12442, 12467, 12498, 12441, 12523, 12501, 12449, 12521, 12483, 12488, 12441,
+ 12501, 12451, 12540, 12488, 12501, 12441, 12483, 12471, 12455, 12523, 12501, 12521, 12531,
+ 12504, 12463, 12479, 12540, 12523, 12504, 12442, 12477, 12504, 12442, 12491, 12498, 12504,
+ 12523, 12484, 12504, 12442, 12531, 12473, 12504, 12442, 12540, 12471, 12441, 12504, 12441,
+ 12540, 12479, 12507, 12442, 12452, 12531, 12488, 12507, 12441, 12523, 12488, 12507, 12531,
+ 12507, 12442, 12531, 12488, 12441, 12507, 12540, 12523, 12507, 12540, 12531, 12510, 12452,
+ 12463, 12525, 12510, 12452, 12523, 12510, 12483, 12495, 12510, 12523, 12463, 12510, 12531,
+ 12471, 12519, 12531, 12511, 12463, 12525, 12531, 12511, 12522, 12511, 12522, 12495, 12441,
+ 12540, 12523, 12513, 12459, 12441, 12513, 12459, 12441, 12488, 12531, 12513, 12540, 12488,
+ 12523, 12516, 12540, 12488, 12441, 12516, 12540, 12523, 12518, 12450, 12531, 12522, 12483,
+ 12488, 12523, 12522, 12521, 12523, 12498, 12442, 12540, 12523, 12540, 12501, 12441, 12523,
+ 12524, 12512, 12524, 12531, 12488, 12465, 12441, 12531, 12527, 12483, 12488, 48, 28857, 49,
+ 28857, 50, 28857, 51, 28857, 52, 28857, 53, 28857, 54, 28857, 55, 28857, 56, 28857, 57, 28857,
+ 49, 48, 28857, 49, 49, 28857, 49, 50, 28857, 49, 51, 28857, 49, 52, 28857, 49, 53, 28857, 49,
+ 54, 28857, 49, 55, 28857, 49, 56, 28857, 49, 57, 28857, 50, 48, 28857, 50, 49, 28857, 50, 50,
+ 28857, 50, 51, 28857, 50, 52, 28857, 104, 80, 97, 100, 97, 65, 85, 98, 97, 114, 111, 86, 112,
+ 99, 100, 109, 100, 109, 50, 100, 109, 51, 73, 85, 24179, 25104, 26157, 21644, 22823, 27491,
+ 26126, 27835, 26666, 24335, 20250, 31038, 112, 65, 110, 65, 956, 65, 109, 65, 107, 65, 75, 66,
+ 77, 66, 71, 66, 99, 97, 108, 107, 99, 97, 108, 112, 70, 110, 70, 956, 70, 956, 103, 109, 103,
+ 107, 103, 72, 122, 107, 72, 122, 77, 72, 122, 71, 72, 122, 84, 72, 122, 956, 108, 109, 108,
+ 100, 108, 107, 108, 102, 109, 110, 109, 956, 109, 109, 109, 99, 109, 107, 109, 109, 109, 50,
+ 99, 109, 50, 109, 50, 107, 109, 50, 109, 109, 51, 99, 109, 51, 109, 51, 107, 109, 51, 109,
+ 8725, 115, 109, 8725, 115, 50, 80, 97, 107, 80, 97, 77, 80, 97, 71, 80, 97, 114, 97, 100, 114,
+ 97, 100, 8725, 115, 114, 97, 100, 8725, 115, 50, 112, 115, 110, 115, 956, 115, 109, 115, 112,
+ 86, 110, 86, 956, 86, 109, 86, 107, 86, 77, 86, 112, 87, 110, 87, 956, 87, 109, 87, 107, 87,
+ 77, 87, 107, 937, 77, 937, 97, 46, 109, 46, 66, 113, 99, 99, 99, 100, 67, 8725, 107, 103, 67,
+ 111, 46, 100, 66, 71, 121, 104, 97, 72, 80, 105, 110, 75, 75, 75, 77, 107, 116, 108, 109, 108,
+ 110, 108, 111, 103, 108, 120, 109, 98, 109, 105, 108, 109, 111, 108, 80, 72, 112, 46, 109, 46,
+ 80, 80, 77, 80, 82, 115, 114, 83, 118, 87, 98, 86, 8725, 109, 65, 8725, 109, 49, 26085, 50,
+ 26085, 51, 26085, 52, 26085, 53, 26085, 54, 26085, 55, 26085, 56, 26085, 57, 26085, 49, 48,
+ 26085, 49, 49, 26085, 49, 50, 26085, 49, 51, 26085, 49, 52, 26085, 49, 53, 26085, 49, 54,
+ 26085, 49, 55, 26085, 49, 56, 26085, 49, 57, 26085, 50, 48, 26085, 50, 49, 26085, 50, 50,
+ 26085, 50, 51, 26085, 50, 52, 26085, 50, 53, 26085, 50, 54, 26085, 50, 55, 26085, 50, 56,
+ 26085, 50, 57, 26085, 51, 48, 26085, 51, 49, 26085, 103, 97, 108, 1098, 1100, 42863, 294, 339,
+ 42791, 43831, 619, 43858, 102, 102, 102, 105, 102, 108, 102, 102, 105, 102, 102, 108, 115, 116,
+ 115, 116, 1396, 1398, 1396, 1381, 1396, 1387, 1406, 1398, 1396, 1389, 1506, 1488, 1491, 1492,
+ 1499, 1500, 1501, 1512, 1514, 43, 1488, 1500, 1649, 1649, 1659, 1659, 1659, 1659, 1662, 1662,
+ 1662, 1662, 1664, 1664, 1664, 1664, 1658, 1658, 1658, 1658, 1663, 1663, 1663, 1663, 1657, 1657,
+ 1657, 1657, 1700, 1700, 1700, 1700, 1702, 1702, 1702, 1702, 1668, 1668, 1668, 1668, 1667, 1667,
+ 1667, 1667, 1670, 1670, 1670, 1670, 1671, 1671, 1671, 1671, 1677, 1677, 1676, 1676, 1678, 1678,
+ 1672, 1672, 1688, 1688, 1681, 1681, 1705, 1705, 1705, 1705, 1711, 1711, 1711, 1711, 1715, 1715,
+ 1715, 1715, 1713, 1713, 1713, 1713, 1722, 1722, 1723, 1723, 1723, 1723, 1749, 1620, 1749, 1620,
+ 1729, 1729, 1729, 1729, 1726, 1726, 1726, 1726, 1746, 1746, 1746, 1620, 1746, 1620, 1709, 1709,
+ 1709, 1709, 1735, 1735, 1734, 1734, 1736, 1736, 1735, 1652, 1739, 1739, 1733, 1733, 1737, 1737,
+ 1744, 1744, 1744, 1744, 1609, 1609, 1610, 1620, 1575, 1610, 1620, 1575, 1610, 1620, 1749, 1610,
+ 1620, 1749, 1610, 1620, 1608, 1610, 1620, 1608, 1610, 1620, 1735, 1610, 1620, 1735, 1610, 1620,
+ 1734, 1610, 1620, 1734, 1610, 1620, 1736, 1610, 1620, 1736, 1610, 1620, 1744, 1610, 1620, 1744,
+ 1610, 1620, 1744, 1610, 1620, 1609, 1610, 1620, 1609, 1610, 1620, 1609, 1740, 1740, 1740, 1740,
+ 1610, 1620, 1580, 1610, 1620, 1581, 1610, 1620, 1605, 1610, 1620, 1609, 1610, 1620, 1610, 1576,
+ 1580, 1576, 1581, 1576, 1582, 1576, 1605, 1576, 1609, 1576, 1610, 1578, 1580, 1578, 1581, 1578,
+ 1582, 1578, 1605, 1578, 1609, 1578, 1610, 1579, 1580, 1579, 1605, 1579, 1609, 1579, 1610, 1580,
+ 1581, 1580, 1605, 1581, 1580, 1581, 1605, 1582, 1580, 1582, 1581, 1582, 1605, 1587, 1580, 1587,
+ 1581, 1587, 1582, 1587, 1605, 1589, 1581, 1589, 1605, 1590, 1580, 1590, 1581, 1590, 1582, 1590,
+ 1605, 1591, 1581, 1591, 1605, 1592, 1605, 1593, 1580, 1593, 1605, 1594, 1580, 1594, 1605, 1601,
+ 1580, 1601, 1581, 1601, 1582, 1601, 1605, 1601, 1609, 1601, 1610, 1602, 1581, 1602, 1605, 1602,
+ 1609, 1602, 1610, 1603, 1575, 1603, 1580, 1603, 1581, 1603, 1582, 1603, 1604, 1603, 1605, 1603,
+ 1609, 1603, 1610, 1604, 1580, 1604, 1581, 1604, 1582, 1604, 1605, 1604, 1609, 1604, 1610, 1605,
+ 1580, 1605, 1581, 1605, 1582, 1605, 1605, 1605, 1609, 1605, 1610, 1606, 1580, 1606, 1581, 1606,
+ 1582, 1606, 1605, 1606, 1609, 1606, 1610, 1607, 1580, 1607, 1605, 1607, 1609, 1607, 1610, 1610,
+ 1580, 1610, 1581, 1610, 1582, 1610, 1605, 1610, 1609, 1610, 1610, 1584, 1648, 1585, 1648, 1609,
+ 1648, 32, 1612, 1617, 32, 1613, 1617, 32, 1614, 1617, 32, 1615, 1617, 32, 1616, 1617, 32, 1617,
+ 1648, 1610, 1620, 1585, 1610, 1620, 1586, 1610, 1620, 1605, 1610, 1620, 1606, 1610, 1620, 1609,
+ 1610, 1620, 1610, 1576, 1585, 1576, 1586, 1576, 1605, 1576, 1606, 1576, 1609, 1576, 1610, 1578,
+ 1585, 1578, 1586, 1578, 1605, 1578, 1606, 1578, 1609, 1578, 1610, 1579, 1585, 1579, 1586, 1579,
+ 1605, 1579, 1606, 1579, 1609, 1579, 1610, 1601, 1609, 1601, 1610, 1602, 1609, 1602, 1610, 1603,
+ 1575, 1603, 1604, 1603, 1605, 1603, 1609, 1603, 1610, 1604, 1605, 1604, 1609, 1604, 1610, 1605,
+ 1575, 1605, 1605, 1606, 1585, 1606, 1586, 1606, 1605, 1606, 1606, 1606, 1609, 1606, 1610, 1609,
+ 1648, 1610, 1585, 1610, 1586, 1610, 1605, 1610, 1606, 1610, 1609, 1610, 1610, 1610, 1620, 1580,
+ 1610, 1620, 1581, 1610, 1620, 1582, 1610, 1620, 1605, 1610, 1620, 1607, 1576, 1580, 1576, 1581,
+ 1576, 1582, 1576, 1605, 1576, 1607, 1578, 1580, 1578, 1581, 1578, 1582, 1578, 1605, 1578, 1607,
+ 1579, 1605, 1580, 1581, 1580, 1605, 1581, 1580, 1581, 1605, 1582, 1580, 1582, 1605, 1587, 1580,
+ 1587, 1581, 1587, 1582, 1587, 1605, 1589, 1581, 1589, 1582, 1589, 1605, 1590, 1580, 1590, 1581,
+ 1590, 1582, 1590, 1605, 1591, 1581, 1592, 1605, 1593, 1580, 1593, 1605, 1594, 1580, 1594, 1605,
+ 1601, 1580, 1601, 1581, 1601, 1582, 1601, 1605, 1602, 1581, 1602, 1605, 1603, 1580, 1603, 1581,
+ 1603, 1582, 1603, 1604, 1603, 1605, 1604, 1580, 1604, 1581, 1604, 1582, 1604, 1605, 1604, 1607,
+ 1605, 1580, 1605, 1581, 1605, 1582, 1605, 1605, 1606, 1580, 1606, 1581, 1606, 1582, 1606, 1605,
+ 1606, 1607, 1607, 1580, 1607, 1605, 1607, 1648, 1610, 1580, 1610, 1581, 1610, 1582, 1610, 1605,
+ 1610, 1607, 1610, 1620, 1605, 1610, 1620, 1607, 1576, 1605, 1576, 1607, 1578, 1605, 1578, 1607,
+ 1579, 1605, 1579, 1607, 1587, 1605, 1587, 1607, 1588, 1605, 1588, 1607, 1603, 1604, 1603, 1605,
+ 1604, 1605, 1606, 1605, 1606, 1607, 1610, 1605, 1610, 1607, 1600, 1614, 1617, 1600, 1615, 1617,
+ 1600, 1616, 1617, 1591, 1609, 1591, 1610, 1593, 1609, 1593, 1610, 1594, 1609, 1594, 1610, 1587,
+ 1609, 1587, 1610, 1588, 1609, 1588, 1610, 1581, 1609, 1581, 1610, 1580, 1609, 1580, 1610, 1582,
+ 1609, 1582, 1610, 1589, 1609, 1589, 1610, 1590, 1609, 1590, 1610, 1588, 1580, 1588, 1581, 1588,
+ 1582, 1588, 1605, 1588, 1585, 1587, 1585, 1589, 1585, 1590, 1585, 1591, 1609, 1591, 1610, 1593,
+ 1609, 1593, 1610, 1594, 1609, 1594, 1610, 1587, 1609, 1587, 1610, 1588, 1609, 1588, 1610, 1581,
+ 1609, 1581, 1610, 1580, 1609, 1580, 1610, 1582, 1609, 1582, 1610, 1589, 1609, 1589, 1610, 1590,
+ 1609, 1590, 1610, 1588, 1580, 1588, 1581, 1588, 1582, 1588, 1605, 1588, 1585, 1587, 1585, 1589,
+ 1585, 1590, 1585, 1588, 1580, 1588, 1581, 1588, 1582, 1588, 1605, 1587, 1607, 1588, 1607, 1591,
+ 1605, 1587, 1580, 1587, 1581, 1587, 1582, 1588, 1580, 1588, 1581, 1588, 1582, 1591, 1605, 1592,
+ 1605, 1575, 1611, 1575, 1611, 1578, 1580, 1605, 1578, 1581, 1580, 1578, 1581, 1580, 1578, 1581,
+ 1605, 1578, 1582, 1605, 1578, 1605, 1580, 1578, 1605, 1581, 1578, 1605, 1582, 1580, 1605, 1581,
+ 1580, 1605, 1581, 1581, 1605, 1610, 1581, 1605, 1609, 1587, 1581, 1580, 1587, 1580, 1581, 1587,
+ 1580, 1609, 1587, 1605, 1581, 1587, 1605, 1581, 1587, 1605, 1580, 1587, 1605, 1605, 1587, 1605,
+ 1605, 1589, 1581, 1581, 1589, 1581, 1581, 1589, 1605, 1605, 1588, 1581, 1605, 1588, 1581, 1605,
+ 1588, 1580, 1610, 1588, 1605, 1582, 1588, 1605, 1582, 1588, 1605, 1605, 1588, 1605, 1605, 1590,
+ 1581, 1609, 1590, 1582, 1605, 1590, 1582, 1605, 1591, 1605, 1581, 1591, 1605, 1581, 1591, 1605,
+ 1605, 1591, 1605, 1610, 1593, 1580, 1605, 1593, 1605, 1605, 1593, 1605, 1605, 1593, 1605, 1609,
+ 1594, 1605, 1605, 1594, 1605, 1610, 1594, 1605, 1609, 1601, 1582, 1605, 1601, 1582, 1605, 1602,
+ 1605, 1581, 1602, 1605, 1605, 1604, 1581, 1605, 1604, 1581, 1610, 1604, 1581, 1609, 1604, 1580,
+ 1580, 1604, 1580, 1580, 1604, 1582, 1605, 1604, 1582, 1605, 1604, 1605, 1581, 1604, 1605, 1581,
+ 1605, 1581, 1580, 1605, 1581, 1605, 1605, 1581, 1610, 1605, 1580, 1581, 1605, 1580, 1605, 1605,
+ 1582, 1580, 1605, 1582, 1605, 1605, 1580, 1582, 1607, 1605, 1580, 1607, 1605, 1605, 1606, 1581,
+ 1605, 1606, 1581, 1609, 1606, 1580, 1605, 1606, 1580, 1605, 1606, 1580, 1609, 1606, 1605, 1610,
+ 1606, 1605, 1609, 1610, 1605, 1605, 1610, 1605, 1605, 1576, 1582, 1610, 1578, 1580, 1610, 1578,
+ 1580, 1609, 1578, 1582, 1610, 1578, 1582, 1609, 1578, 1605, 1610, 1578, 1605, 1609, 1580, 1605,
+ 1610, 1580, 1581, 1609, 1580, 1605, 1609, 1587, 1582, 1609, 1589, 1581, 1610, 1588, 1581, 1610,
+ 1590, 1581, 1610, 1604, 1580, 1610, 1604, 1605, 1610, 1610, 1581, 1610, 1610, 1580, 1610, 1610,
+ 1605, 1610, 1605, 1605, 1610, 1602, 1605, 1610, 1606, 1581, 1610, 1602, 1605, 1581, 1604, 1581,
+ 1605, 1593, 1605, 1610, 1603, 1605, 1610, 1606, 1580, 1581, 1605, 1582, 1610, 1604, 1580, 1605,
+ 1603, 1605, 1605, 1604, 1580, 1605, 1606, 1580, 1581, 1580, 1581, 1610, 1581, 1580, 1610, 1605,
+ 1580, 1610, 1601, 1605, 1610, 1576, 1581, 1610, 1603, 1605, 1605, 1593, 1580, 1605, 1589, 1605,
+ 1605, 1587, 1582, 1610, 1606, 1580, 1610, 1589, 1604, 1746, 1602, 1604, 1746, 1575, 1604, 1604,
+ 1607, 1575, 1603, 1576, 1585, 1605, 1581, 1605, 1583, 1589, 1604, 1593, 1605, 1585, 1587, 1608,
+ 1604, 1593, 1604, 1610, 1607, 1608, 1587, 1604, 1605, 1589, 1604, 1609, 1589, 1604, 1609, 32,
+ 1575, 1604, 1604, 1607, 32, 1593, 1604, 1610, 1607, 32, 1608, 1587, 1604, 1605, 1580, 1604, 32,
+ 1580, 1604, 1575, 1604, 1607, 1585, 1740, 1575, 1604, 44, 12289, 12290, 58, 59, 33, 63, 12310,
+ 12311, 46, 46, 46, 46, 46, 8212, 8211, 95, 95, 40, 41, 123, 125, 12308, 12309, 12304, 12305,
+ 12298, 12299, 12296, 12297, 12300, 12301, 12302, 12303, 91, 93, 32, 773, 32, 773, 32, 773, 32,
+ 773, 95, 95, 95, 44, 12289, 46, 59, 58, 63, 33, 8212, 40, 41, 123, 125, 12308, 12309, 35, 38,
+ 42, 43, 45, 60, 62, 61, 92, 36, 37, 64, 32, 1611, 1600, 1611, 32, 1612, 32, 1613, 32, 1614,
+ 1600, 1614, 32, 1615, 1600, 1615, 32, 1616, 1600, 1616, 32, 1617, 1600, 1617, 32, 1618, 1600,
+ 1618, 1569, 1575, 1619, 1575, 1619, 1575, 1620, 1575, 1620, 1608, 1620, 1608, 1620, 1575, 1621,
+ 1575, 1621, 1610, 1620, 1610, 1620, 1610, 1620, 1610, 1620, 1575, 1575, 1576, 1576, 1576, 1576,
+ 1577, 1577, 1578, 1578, 1578, 1578, 1579, 1579, 1579, 1579, 1580, 1580, 1580, 1580, 1581, 1581,
+ 1581, 1581, 1582, 1582, 1582, 1582, 1583, 1583, 1584, 1584, 1585, 1585, 1586, 1586, 1587, 1587,
+ 1587, 1587, 1588, 1588, 1588, 1588, 1589, 1589, 1589, 1589, 1590, 1590, 1590, 1590, 1591, 1591,
+ 1591, 1591, 1592, 1592, 1592, 1592, 1593, 1593, 1593, 1593, 1594, 1594, 1594, 1594, 1601, 1601,
+ 1601, 1601, 1602, 1602, 1602, 1602, 1603, 1603, 1603, 1603, 1604, 1604, 1604, 1604, 1605, 1605,
+ 1605, 1605, 1606, 1606, 1606, 1606, 1607, 1607, 1607, 1607, 1608, 1608, 1609, 1609, 1610, 1610,
+ 1610, 1610, 1604, 1575, 1619, 1604, 1575, 1619, 1604, 1575, 1620, 1604, 1575, 1620, 1604, 1575,
+ 1621, 1604, 1575, 1621, 1604, 1575, 1604, 1575, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
+ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68,
+ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92,
+ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112,
+ 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 10629, 10630, 12290,
+ 12300, 12301, 12289, 12539, 12530, 12449, 12451, 12453, 12455, 12457, 12515, 12517, 12519,
+ 12483, 12540, 12450, 12452, 12454, 12456, 12458, 12459, 12461, 12463, 12465, 12467, 12469,
+ 12471, 12473, 12475, 12477, 12479, 12481, 12484, 12486, 12488, 12490, 12491, 12492, 12493,
+ 12494, 12495, 12498, 12501, 12504, 12507, 12510, 12511, 12512, 12513, 12514, 12516, 12518,
+ 12520, 12521, 12522, 12523, 12524, 12525, 12527, 12531, 12441, 12442, 4448, 4352, 4353, 4522,
+ 4354, 4524, 4525, 4355, 4356, 4357, 4528, 4529, 4530, 4531, 4532, 4533, 4378, 4358, 4359, 4360,
+ 4385, 4361, 4362, 4363, 4364, 4365, 4366, 4367, 4368, 4369, 4370, 4449, 4450, 4451, 4452, 4453,
+ 4454, 4455, 4456, 4457, 4458, 4459, 4460, 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, 4469,
+ 162, 163, 172, 32, 772, 166, 165, 8361, 9474, 8592, 8593, 8594, 8595, 9632, 9675, 65, 66, 67,
+ 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97,
+ 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,
+ 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80,
+ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108,
+ 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70,
+ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99,
+ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
+ 119, 120, 121, 122, 65, 67, 68, 71, 74, 75, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 97,
+ 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 112, 113, 114, 115, 116, 117, 118, 119,
+ 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84,
+ 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111,
+ 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 68, 69, 70, 71, 74, 75, 76, 77,
+ 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107,
+ 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 68, 69, 70,
+ 71, 73, 74, 75, 76, 77, 79, 83, 84, 85, 86, 87, 88, 89, 97, 98, 99, 100, 101, 102, 103, 104,
+ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65,
+ 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
+ 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,
+ 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107,
+ 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69,
+ 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99,
+ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,
+ 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83,
+ 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110,
+ 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73,
+ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102,
+ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121,
+ 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86,
+ 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112,
+ 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 305, 567, 913, 914, 915, 916, 917, 918, 919,
+ 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934, 935, 936, 937, 8711,
+ 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963,
+ 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 913, 914, 915, 916, 917, 918,
+ 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934, 935, 936, 937,
+ 8711, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962,
+ 963, 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 913, 914, 915, 916, 917,
+ 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934, 935, 936,
+ 937, 8711, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961,
+ 962, 963, 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 913, 914, 915, 916,
+ 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934, 935,
+ 936, 937, 8711, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960,
+ 961, 962, 963, 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 913, 914, 915,
+ 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934,
+ 935, 936, 937, 8711, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959,
+ 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 988, 989,
+ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 49, 50, 51,
+ 52, 53, 54, 55, 56, 57, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 49, 50, 51, 52, 53, 54, 55,
+ 56, 57, 1575, 1576, 1580, 1583, 1608, 1586, 1581, 1591, 1610, 1603, 1604, 1605, 1606, 1587,
+ 1593, 1601, 1589, 1602, 1585, 1588, 1578, 1579, 1582, 1584, 1590, 1592, 1594, 1646, 1722, 1697,
+ 1647, 1576, 1580, 1607, 1581, 1610, 1603, 1604, 1605, 1606, 1587, 1593, 1601, 1589, 1602, 1588,
+ 1578, 1579, 1582, 1590, 1594, 1580, 1581, 1610, 1604, 1606, 1587, 1593, 1589, 1602, 1588, 1582,
+ 1590, 1594, 1722, 1647, 1576, 1580, 1607, 1581, 1591, 1610, 1603, 1605, 1606, 1587, 1593, 1601,
+ 1589, 1602, 1588, 1578, 1579, 1582, 1590, 1592, 1594, 1646, 1697, 1575, 1576, 1580, 1583, 1607,
+ 1608, 1586, 1581, 1591, 1610, 1604, 1605, 1606, 1587, 1593, 1601, 1589, 1602, 1585, 1588, 1578,
+ 1579, 1582, 1584, 1590, 1592, 1594, 1576, 1580, 1583, 1608, 1586, 1581, 1591, 1610, 1604, 1605,
+ 1606, 1587, 1593, 1601, 1589, 1602, 1585, 1588, 1578, 1579, 1582, 1584, 1590, 1592, 1594, 48,
+ 46, 48, 44, 49, 44, 50, 44, 51, 44, 52, 44, 53, 44, 54, 44, 55, 44, 56, 44, 57, 44, 40, 65, 41,
+ 40, 66, 41, 40, 67, 41, 40, 68, 41, 40, 69, 41, 40, 70, 41, 40, 71, 41, 40, 72, 41, 40, 73, 41,
+ 40, 74, 41, 40, 75, 41, 40, 76, 41, 40, 77, 41, 40, 78, 41, 40, 79, 41, 40, 80, 41, 40, 81, 41,
+ 40, 82, 41, 40, 83, 41, 40, 84, 41, 40, 85, 41, 40, 86, 41, 40, 87, 41, 40, 88, 41, 40, 89, 41,
+ 40, 90, 41, 12308, 83, 12309, 67, 82, 67, 68, 87, 90, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74,
+ 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 72, 86, 77, 86, 83, 68, 83, 83,
+ 80, 80, 86, 87, 67, 77, 67, 77, 68, 68, 74, 12411, 12363, 12467, 12467, 12469, 25163, 23383,
+ 21452, 12486, 12441, 20108, 22810, 35299, 22825, 20132, 26144, 28961, 26009, 21069, 24460,
+ 20877, 26032, 21021, 32066, 29983, 36009, 22768, 21561, 28436, 25237, 25429, 19968, 19977,
+ 36938, 24038, 20013, 21491, 25351, 36208, 25171, 31105, 31354, 21512, 28288, 26377, 26376,
+ 30003, 21106, 21942, 37197, 12308, 26412, 12309, 12308, 19977, 12309, 12308, 20108, 12309,
+ 12308, 23433, 12309, 12308, 28857, 12309, 12308, 25171, 12309, 12308, 30423, 12309, 12308,
+ 21213, 12309, 12308, 25943, 12309, 24471, 21487,
+};
+
+// cccRanges are (lo, hi, class) triples of the canonical combining class, sorted.
+inline constexpr uint32_t k_ccc_ranges[] = {
+ 768, 788, 230, 789, 789, 232, 790, 793, 220, 794, 794, 232, 795, 795, 216, 796, 800, 220, 801,
+ 802, 202, 803, 806, 220, 807, 808, 202, 809, 819, 220, 820, 824, 1, 825, 828, 220, 829, 831,
+ 230, 834, 834, 230, 837, 837, 240, 838, 838, 230, 839, 841, 220, 842, 844, 230, 845, 846, 220,
+ 848, 850, 230, 851, 854, 220, 855, 855, 230, 856, 856, 232, 857, 858, 220, 859, 859, 230, 860,
+ 860, 233, 861, 862, 234, 863, 863, 233, 864, 865, 234, 866, 866, 233, 867, 879, 230, 1155,
+ 1159, 230, 1425, 1425, 220, 1426, 1429, 230, 1430, 1430, 220, 1431, 1433, 230, 1434, 1434, 222,
+ 1435, 1435, 220, 1436, 1441, 230, 1442, 1447, 220, 1448, 1449, 230, 1450, 1450, 220, 1451,
+ 1452, 230, 1453, 1453, 222, 1454, 1454, 228, 1455, 1455, 230, 1456, 1456, 10, 1457, 1457, 11,
+ 1458, 1458, 12, 1459, 1459, 13, 1460, 1460, 14, 1461, 1461, 15, 1462, 1462, 16, 1463, 1463, 17,
+ 1464, 1464, 18, 1465, 1466, 19, 1467, 1467, 20, 1468, 1468, 21, 1469, 1469, 22, 1471, 1471, 23,
+ 1473, 1473, 24, 1474, 1474, 25, 1476, 1476, 230, 1477, 1477, 220, 1479, 1479, 18, 1552, 1559,
+ 230, 1560, 1560, 30, 1561, 1561, 31, 1562, 1562, 32, 1611, 1611, 27, 1612, 1612, 28, 1613,
+ 1613, 29, 1614, 1614, 30, 1615, 1615, 31, 1616, 1616, 32, 1617, 1617, 33, 1618, 1618, 34, 1619,
+ 1620, 230, 1621, 1622, 220, 1623, 1627, 230, 1628, 1628, 220, 1629, 1630, 230, 1631, 1631, 220,
+ 1648, 1648, 35, 1750, 1756, 230, 1759, 1762, 230, 1763, 1763, 220, 1764, 1764, 230, 1767, 1768,
+ 230, 1770, 1770, 220, 1771, 1772, 230, 1773, 1773, 220, 1809, 1809, 36, 1840, 1840, 230, 1841,
+ 1841, 220, 1842, 1843, 230, 1844, 1844, 220, 1845, 1846, 230, 1847, 1849, 220, 1850, 1850, 230,
+ 1851, 1852, 220, 1853, 1853, 230, 1854, 1854, 220, 1855, 1857, 230, 1858, 1858, 220, 1859,
+ 1859, 230, 1860, 1860, 220, 1861, 1861, 230, 1862, 1862, 220, 1863, 1863, 230, 1864, 1864, 220,
+ 1865, 1866, 230, 2027, 2033, 230, 2034, 2034, 220, 2035, 2035, 230, 2070, 2073, 230, 2075,
+ 2083, 230, 2085, 2087, 230, 2089, 2093, 230, 2137, 2139, 220, 2260, 2273, 230, 2275, 2275, 220,
+ 2276, 2277, 230, 2278, 2278, 220, 2279, 2280, 230, 2281, 2281, 220, 2282, 2284, 230, 2285,
+ 2287, 220, 2288, 2288, 27, 2289, 2289, 28, 2290, 2290, 29, 2291, 2293, 230, 2294, 2294, 220,
+ 2295, 2296, 230, 2297, 2298, 220, 2299, 2303, 230, 2364, 2364, 7, 2381, 2381, 9, 2385, 2385,
+ 230, 2386, 2386, 220, 2387, 2388, 230, 2492, 2492, 7, 2509, 2509, 9, 2620, 2620, 7, 2637, 2637,
+ 9, 2748, 2748, 7, 2765, 2765, 9, 2876, 2876, 7, 2893, 2893, 9, 3021, 3021, 9, 3149, 3149, 9,
+ 3157, 3157, 84, 3158, 3158, 91, 3260, 3260, 7, 3277, 3277, 9, 3405, 3405, 9, 3530, 3530, 9,
+ 3640, 3641, 103, 3642, 3642, 9, 3656, 3659, 107, 3768, 3769, 118, 3784, 3787, 122, 3864, 3865,
+ 220, 3893, 3893, 220, 3895, 3895, 220, 3897, 3897, 216, 3953, 3953, 129, 3954, 3954, 130, 3956,
+ 3956, 132, 3962, 3965, 130, 3968, 3968, 130, 3970, 3971, 230, 3972, 3972, 9, 3974, 3975, 230,
+ 4038, 4038, 220, 4151, 4151, 7, 4153, 4154, 9, 4237, 4237, 220, 4957, 4959, 230, 5908, 5908, 9,
+ 5940, 5940, 9, 6098, 6098, 9, 6109, 6109, 230, 6313, 6313, 228, 6457, 6457, 222, 6458, 6458,
+ 230, 6459, 6459, 220, 6679, 6679, 230, 6680, 6680, 220, 6752, 6752, 9, 6773, 6780, 230, 6783,
+ 6783, 220, 6832, 6836, 230, 6837, 6842, 220, 6843, 6844, 230, 6845, 6845, 220, 6964, 6964, 7,
+ 6980, 6980, 9, 7019, 7019, 230, 7020, 7020, 220, 7021, 7027, 230, 7082, 7083, 9, 7142, 7142, 7,
+ 7154, 7155, 9, 7223, 7223, 7, 7376, 7378, 230, 7380, 7380, 1, 7381, 7385, 220, 7386, 7387, 230,
+ 7388, 7391, 220, 7392, 7392, 230, 7394, 7400, 1, 7405, 7405, 220, 7412, 7412, 230, 7416, 7417,
+ 230, 7616, 7617, 230, 7618, 7618, 220, 7619, 7625, 230, 7626, 7626, 220, 7627, 7628, 230, 7629,
+ 7629, 234, 7630, 7630, 214, 7631, 7631, 220, 7632, 7632, 202, 7633, 7669, 230, 7675, 7675, 230,
+ 7676, 7676, 233, 7677, 7677, 220, 7678, 7678, 230, 7679, 7679, 220, 8400, 8401, 230, 8402,
+ 8403, 1, 8404, 8407, 230, 8408, 8410, 1, 8411, 8412, 230, 8417, 8417, 230, 8421, 8422, 1, 8423,
+ 8423, 230, 8424, 8424, 220, 8425, 8425, 230, 8426, 8427, 1, 8428, 8431, 220, 8432, 8432, 230,
+ 11503, 11505, 230, 11647, 11647, 9, 11744, 11775, 230, 12330, 12330, 218, 12331, 12331, 228,
+ 12332, 12332, 232, 12333, 12333, 222, 12334, 12335, 224, 12441, 12442, 8, 42607, 42607, 230,
+ 42612, 42621, 230, 42654, 42655, 230, 42736, 42737, 230, 43014, 43014, 9, 43204, 43204, 9,
+ 43232, 43249, 230, 43307, 43309, 220, 43347, 43347, 9, 43443, 43443, 7, 43456, 43456, 9, 43696,
+ 43696, 230, 43698, 43699, 230, 43700, 43700, 220, 43703, 43704, 230, 43710, 43711, 230, 43713,
+ 43713, 230, 43766, 43766, 9, 44013, 44013, 9, 64286, 64286, 26, 65056, 65062, 230, 65063,
+ 65069, 220, 65070, 65071, 230, 66045, 66045, 220, 66272, 66272, 220, 66422, 66426, 230, 68109,
+ 68109, 220, 68111, 68111, 230, 68152, 68152, 230, 68153, 68153, 1, 68154, 68154, 220, 68159,
+ 68159, 9, 68325, 68325, 230, 68326, 68326, 220, 69702, 69702, 9, 69759, 69759, 9, 69817, 69817,
+ 9, 69818, 69818, 7, 69888, 69890, 230, 69939, 69940, 9, 70003, 70003, 7, 70080, 70080, 9,
+ 70090, 70090, 7, 70197, 70197, 9, 70198, 70198, 7, 70377, 70377, 7, 70378, 70378, 9, 70460,
+ 70460, 7, 70477, 70477, 9, 70502, 70508, 230, 70512, 70516, 230, 70722, 70722, 9, 70726, 70726,
+ 7, 70850, 70850, 9, 70851, 70851, 7, 71103, 71103, 9, 71104, 71104, 7, 71231, 71231, 9, 71350,
+ 71350, 9, 71351, 71351, 7, 71467, 71467, 9, 72767, 72767, 9, 92912, 92916, 1, 92976, 92982,
+ 230, 113822, 113822, 1, 119141, 119142, 216, 119143, 119145, 1, 119149, 119149, 226, 119150,
+ 119154, 216, 119163, 119170, 220, 119173, 119177, 230, 119178, 119179, 220, 119210, 119213,
+ 230, 119362, 119364, 230, 122880, 122886, 230, 122888, 122904, 230, 122907, 122913, 230,
+ 122915, 122916, 230, 122918, 122922, 230, 125136, 125142, 220, 125252, 125257, 230, 125258,
+ 125258, 7,
+};
+
+// composeKeys are (first<<32 | second) of each primary composition pair, sorted;
+// composeValues is the composite.
+inline constexpr uint64_t k_compose_keys[] = {
+ 0x3c00000338ULL, 0x3d00000338ULL, 0x3e00000338ULL, 0x4100000300ULL, 0x4100000301ULL,
+ 0x4100000302ULL, 0x4100000303ULL, 0x4100000304ULL, 0x4100000306ULL, 0x4100000307ULL,
+ 0x4100000308ULL, 0x4100000309ULL, 0x410000030aULL, 0x410000030cULL, 0x410000030fULL,
+ 0x4100000311ULL, 0x4100000323ULL, 0x4100000325ULL, 0x4100000328ULL, 0x4200000307ULL,
+ 0x4200000323ULL, 0x4200000331ULL, 0x4300000301ULL, 0x4300000302ULL, 0x4300000307ULL,
+ 0x430000030cULL, 0x4300000327ULL, 0x4400000307ULL, 0x440000030cULL, 0x4400000323ULL,
+ 0x4400000327ULL, 0x440000032dULL, 0x4400000331ULL, 0x4500000300ULL, 0x4500000301ULL,
+ 0x4500000302ULL, 0x4500000303ULL, 0x4500000304ULL, 0x4500000306ULL, 0x4500000307ULL,
+ 0x4500000308ULL, 0x4500000309ULL, 0x450000030cULL, 0x450000030fULL, 0x4500000311ULL,
+ 0x4500000323ULL, 0x4500000327ULL, 0x4500000328ULL, 0x450000032dULL, 0x4500000330ULL,
+ 0x4600000307ULL, 0x4700000301ULL, 0x4700000302ULL, 0x4700000304ULL, 0x4700000306ULL,
+ 0x4700000307ULL, 0x470000030cULL, 0x4700000327ULL, 0x4800000302ULL, 0x4800000307ULL,
+ 0x4800000308ULL, 0x480000030cULL, 0x4800000323ULL, 0x4800000327ULL, 0x480000032eULL,
+ 0x4900000300ULL, 0x4900000301ULL, 0x4900000302ULL, 0x4900000303ULL, 0x4900000304ULL,
+ 0x4900000306ULL, 0x4900000307ULL, 0x4900000308ULL, 0x4900000309ULL, 0x490000030cULL,
+ 0x490000030fULL, 0x4900000311ULL, 0x4900000323ULL, 0x4900000328ULL, 0x4900000330ULL,
+ 0x4a00000302ULL, 0x4b00000301ULL, 0x4b0000030cULL, 0x4b00000323ULL, 0x4b00000327ULL,
+ 0x4b00000331ULL, 0x4c00000301ULL, 0x4c0000030cULL, 0x4c00000323ULL, 0x4c00000327ULL,
+ 0x4c0000032dULL, 0x4c00000331ULL, 0x4d00000301ULL, 0x4d00000307ULL, 0x4d00000323ULL,
+ 0x4e00000300ULL, 0x4e00000301ULL, 0x4e00000303ULL, 0x4e00000307ULL, 0x4e0000030cULL,
+ 0x4e00000323ULL, 0x4e00000327ULL, 0x4e0000032dULL, 0x4e00000331ULL, 0x4f00000300ULL,
+ 0x4f00000301ULL, 0x4f00000302ULL, 0x4f00000303ULL, 0x4f00000304ULL, 0x4f00000306ULL,
+ 0x4f00000307ULL, 0x4f00000308ULL, 0x4f00000309ULL, 0x4f0000030bULL, 0x4f0000030cULL,
+ 0x4f0000030fULL, 0x4f00000311ULL, 0x4f0000031bULL, 0x4f00000323ULL, 0x4f00000328ULL,
+ 0x5000000301ULL, 0x5000000307ULL, 0x5200000301ULL, 0x5200000307ULL, 0x520000030cULL,
+ 0x520000030fULL, 0x5200000311ULL, 0x5200000323ULL, 0x5200000327ULL, 0x5200000331ULL,
+ 0x5300000301ULL, 0x5300000302ULL, 0x5300000307ULL, 0x530000030cULL, 0x5300000323ULL,
+ 0x5300000326ULL, 0x5300000327ULL, 0x5400000307ULL, 0x540000030cULL, 0x5400000323ULL,
+ 0x5400000326ULL, 0x5400000327ULL, 0x540000032dULL, 0x5400000331ULL, 0x5500000300ULL,
+ 0x5500000301ULL, 0x5500000302ULL, 0x5500000303ULL, 0x5500000304ULL, 0x5500000306ULL,
+ 0x5500000308ULL, 0x5500000309ULL, 0x550000030aULL, 0x550000030bULL, 0x550000030cULL,
+ 0x550000030fULL, 0x5500000311ULL, 0x550000031bULL, 0x5500000323ULL, 0x5500000324ULL,
+ 0x5500000328ULL, 0x550000032dULL, 0x5500000330ULL, 0x5600000303ULL, 0x5600000323ULL,
+ 0x5700000300ULL, 0x5700000301ULL, 0x5700000302ULL, 0x5700000307ULL, 0x5700000308ULL,
+ 0x5700000323ULL, 0x5800000307ULL, 0x5800000308ULL, 0x5900000300ULL, 0x5900000301ULL,
+ 0x5900000302ULL, 0x5900000303ULL, 0x5900000304ULL, 0x5900000307ULL, 0x5900000308ULL,
+ 0x5900000309ULL, 0x5900000323ULL, 0x5a00000301ULL, 0x5a00000302ULL, 0x5a00000307ULL,
+ 0x5a0000030cULL, 0x5a00000323ULL, 0x5a00000331ULL, 0x6100000300ULL, 0x6100000301ULL,
+ 0x6100000302ULL, 0x6100000303ULL, 0x6100000304ULL, 0x6100000306ULL, 0x6100000307ULL,
+ 0x6100000308ULL, 0x6100000309ULL, 0x610000030aULL, 0x610000030cULL, 0x610000030fULL,
+ 0x6100000311ULL, 0x6100000323ULL, 0x6100000325ULL, 0x6100000328ULL, 0x6200000307ULL,
+ 0x6200000323ULL, 0x6200000331ULL, 0x6300000301ULL, 0x6300000302ULL, 0x6300000307ULL,
+ 0x630000030cULL, 0x6300000327ULL, 0x6400000307ULL, 0x640000030cULL, 0x6400000323ULL,
+ 0x6400000327ULL, 0x640000032dULL, 0x6400000331ULL, 0x6500000300ULL, 0x6500000301ULL,
+ 0x6500000302ULL, 0x6500000303ULL, 0x6500000304ULL, 0x6500000306ULL, 0x6500000307ULL,
+ 0x6500000308ULL, 0x6500000309ULL, 0x650000030cULL, 0x650000030fULL, 0x6500000311ULL,
+ 0x6500000323ULL, 0x6500000327ULL, 0x6500000328ULL, 0x650000032dULL, 0x6500000330ULL,
+ 0x6600000307ULL, 0x6700000301ULL, 0x6700000302ULL, 0x6700000304ULL, 0x6700000306ULL,
+ 0x6700000307ULL, 0x670000030cULL, 0x6700000327ULL, 0x6800000302ULL, 0x6800000307ULL,
+ 0x6800000308ULL, 0x680000030cULL, 0x6800000323ULL, 0x6800000327ULL, 0x680000032eULL,
+ 0x6800000331ULL, 0x6900000300ULL, 0x6900000301ULL, 0x6900000302ULL, 0x6900000303ULL,
+ 0x6900000304ULL, 0x6900000306ULL, 0x6900000308ULL, 0x6900000309ULL, 0x690000030cULL,
+ 0x690000030fULL, 0x6900000311ULL, 0x6900000323ULL, 0x6900000328ULL, 0x6900000330ULL,
+ 0x6a00000302ULL, 0x6a0000030cULL, 0x6b00000301ULL, 0x6b0000030cULL, 0x6b00000323ULL,
+ 0x6b00000327ULL, 0x6b00000331ULL, 0x6c00000301ULL, 0x6c0000030cULL, 0x6c00000323ULL,
+ 0x6c00000327ULL, 0x6c0000032dULL, 0x6c00000331ULL, 0x6d00000301ULL, 0x6d00000307ULL,
+ 0x6d00000323ULL, 0x6e00000300ULL, 0x6e00000301ULL, 0x6e00000303ULL, 0x6e00000307ULL,
+ 0x6e0000030cULL, 0x6e00000323ULL, 0x6e00000327ULL, 0x6e0000032dULL, 0x6e00000331ULL,
+ 0x6f00000300ULL, 0x6f00000301ULL, 0x6f00000302ULL, 0x6f00000303ULL, 0x6f00000304ULL,
+ 0x6f00000306ULL, 0x6f00000307ULL, 0x6f00000308ULL, 0x6f00000309ULL, 0x6f0000030bULL,
+ 0x6f0000030cULL, 0x6f0000030fULL, 0x6f00000311ULL, 0x6f0000031bULL, 0x6f00000323ULL,
+ 0x6f00000328ULL, 0x7000000301ULL, 0x7000000307ULL, 0x7200000301ULL, 0x7200000307ULL,
+ 0x720000030cULL, 0x720000030fULL, 0x7200000311ULL, 0x7200000323ULL, 0x7200000327ULL,
+ 0x7200000331ULL, 0x7300000301ULL, 0x7300000302ULL, 0x7300000307ULL, 0x730000030cULL,
+ 0x7300000323ULL, 0x7300000326ULL, 0x7300000327ULL, 0x7400000307ULL, 0x7400000308ULL,
+ 0x740000030cULL, 0x7400000323ULL, 0x7400000326ULL, 0x7400000327ULL, 0x740000032dULL,
+ 0x7400000331ULL, 0x7500000300ULL, 0x7500000301ULL, 0x7500000302ULL, 0x7500000303ULL,
+ 0x7500000304ULL, 0x7500000306ULL, 0x7500000308ULL, 0x7500000309ULL, 0x750000030aULL,
+ 0x750000030bULL, 0x750000030cULL, 0x750000030fULL, 0x7500000311ULL, 0x750000031bULL,
+ 0x7500000323ULL, 0x7500000324ULL, 0x7500000328ULL, 0x750000032dULL, 0x7500000330ULL,
+ 0x7600000303ULL, 0x7600000323ULL, 0x7700000300ULL, 0x7700000301ULL, 0x7700000302ULL,
+ 0x7700000307ULL, 0x7700000308ULL, 0x770000030aULL, 0x7700000323ULL, 0x7800000307ULL,
+ 0x7800000308ULL, 0x7900000300ULL, 0x7900000301ULL, 0x7900000302ULL, 0x7900000303ULL,
+ 0x7900000304ULL, 0x7900000307ULL, 0x7900000308ULL, 0x7900000309ULL, 0x790000030aULL,
+ 0x7900000323ULL, 0x7a00000301ULL, 0x7a00000302ULL, 0x7a00000307ULL, 0x7a0000030cULL,
+ 0x7a00000323ULL, 0x7a00000331ULL, 0xa800000300ULL, 0xa800000301ULL, 0xa800000342ULL,
+ 0xc200000300ULL, 0xc200000301ULL, 0xc200000303ULL, 0xc200000309ULL, 0xc400000304ULL,
+ 0xc500000301ULL, 0xc600000301ULL, 0xc600000304ULL, 0xc700000301ULL, 0xca00000300ULL,
+ 0xca00000301ULL, 0xca00000303ULL, 0xca00000309ULL, 0xcf00000301ULL, 0xd400000300ULL,
+ 0xd400000301ULL, 0xd400000303ULL, 0xd400000309ULL, 0xd500000301ULL, 0xd500000304ULL,
+ 0xd500000308ULL, 0xd600000304ULL, 0xd800000301ULL, 0xdc00000300ULL, 0xdc00000301ULL,
+ 0xdc00000304ULL, 0xdc0000030cULL, 0xe200000300ULL, 0xe200000301ULL, 0xe200000303ULL,
+ 0xe200000309ULL, 0xe400000304ULL, 0xe500000301ULL, 0xe600000301ULL, 0xe600000304ULL,
+ 0xe700000301ULL, 0xea00000300ULL, 0xea00000301ULL, 0xea00000303ULL, 0xea00000309ULL,
+ 0xef00000301ULL, 0xf400000300ULL, 0xf400000301ULL, 0xf400000303ULL, 0xf400000309ULL,
+ 0xf500000301ULL, 0xf500000304ULL, 0xf500000308ULL, 0xf600000304ULL, 0xf800000301ULL,
+ 0xfc00000300ULL, 0xfc00000301ULL, 0xfc00000304ULL, 0xfc0000030cULL, 0x10200000300ULL,
+ 0x10200000301ULL, 0x10200000303ULL, 0x10200000309ULL, 0x10300000300ULL, 0x10300000301ULL,
+ 0x10300000303ULL, 0x10300000309ULL, 0x11200000300ULL, 0x11200000301ULL, 0x11300000300ULL,
+ 0x11300000301ULL, 0x14c00000300ULL, 0x14c00000301ULL, 0x14d00000300ULL, 0x14d00000301ULL,
+ 0x15a00000307ULL, 0x15b00000307ULL, 0x16000000307ULL, 0x16100000307ULL, 0x16800000301ULL,
+ 0x16900000301ULL, 0x16a00000308ULL, 0x16b00000308ULL, 0x17f00000307ULL, 0x1a000000300ULL,
+ 0x1a000000301ULL, 0x1a000000303ULL, 0x1a000000309ULL, 0x1a000000323ULL, 0x1a100000300ULL,
+ 0x1a100000301ULL, 0x1a100000303ULL, 0x1a100000309ULL, 0x1a100000323ULL, 0x1af00000300ULL,
+ 0x1af00000301ULL, 0x1af00000303ULL, 0x1af00000309ULL, 0x1af00000323ULL, 0x1b000000300ULL,
+ 0x1b000000301ULL, 0x1b000000303ULL, 0x1b000000309ULL, 0x1b000000323ULL, 0x1b70000030cULL,
+ 0x1ea00000304ULL, 0x1eb00000304ULL, 0x22600000304ULL, 0x22700000304ULL, 0x22800000306ULL,
+ 0x22900000306ULL, 0x22e00000304ULL, 0x22f00000304ULL, 0x2920000030cULL, 0x39100000300ULL,
+ 0x39100000301ULL, 0x39100000304ULL, 0x39100000306ULL, 0x39100000313ULL, 0x39100000314ULL,
+ 0x39100000345ULL, 0x39500000300ULL, 0x39500000301ULL, 0x39500000313ULL, 0x39500000314ULL,
+ 0x39700000300ULL, 0x39700000301ULL, 0x39700000313ULL, 0x39700000314ULL, 0x39700000345ULL,
+ 0x39900000300ULL, 0x39900000301ULL, 0x39900000304ULL, 0x39900000306ULL, 0x39900000308ULL,
+ 0x39900000313ULL, 0x39900000314ULL, 0x39f00000300ULL, 0x39f00000301ULL, 0x39f00000313ULL,
+ 0x39f00000314ULL, 0x3a100000314ULL, 0x3a500000300ULL, 0x3a500000301ULL, 0x3a500000304ULL,
+ 0x3a500000306ULL, 0x3a500000308ULL, 0x3a500000314ULL, 0x3a900000300ULL, 0x3a900000301ULL,
+ 0x3a900000313ULL, 0x3a900000314ULL, 0x3a900000345ULL, 0x3ac00000345ULL, 0x3ae00000345ULL,
+ 0x3b100000300ULL, 0x3b100000301ULL, 0x3b100000304ULL, 0x3b100000306ULL, 0x3b100000313ULL,
+ 0x3b100000314ULL, 0x3b100000342ULL, 0x3b100000345ULL, 0x3b500000300ULL, 0x3b500000301ULL,
+ 0x3b500000313ULL, 0x3b500000314ULL, 0x3b700000300ULL, 0x3b700000301ULL, 0x3b700000313ULL,
+ 0x3b700000314ULL, 0x3b700000342ULL, 0x3b700000345ULL, 0x3b900000300ULL, 0x3b900000301ULL,
+ 0x3b900000304ULL, 0x3b900000306ULL, 0x3b900000308ULL, 0x3b900000313ULL, 0x3b900000314ULL,
+ 0x3b900000342ULL, 0x3bf00000300ULL, 0x3bf00000301ULL, 0x3bf00000313ULL, 0x3bf00000314ULL,
+ 0x3c100000313ULL, 0x3c100000314ULL, 0x3c500000300ULL, 0x3c500000301ULL, 0x3c500000304ULL,
+ 0x3c500000306ULL, 0x3c500000308ULL, 0x3c500000313ULL, 0x3c500000314ULL, 0x3c500000342ULL,
+ 0x3c900000300ULL, 0x3c900000301ULL, 0x3c900000313ULL, 0x3c900000314ULL, 0x3c900000342ULL,
+ 0x3c900000345ULL, 0x3ca00000300ULL, 0x3ca00000301ULL, 0x3ca00000342ULL, 0x3cb00000300ULL,
+ 0x3cb00000301ULL, 0x3cb00000342ULL, 0x3ce00000345ULL, 0x3d200000301ULL, 0x3d200000308ULL,
+ 0x40600000308ULL, 0x41000000306ULL, 0x41000000308ULL, 0x41300000301ULL, 0x41500000300ULL,
+ 0x41500000306ULL, 0x41500000308ULL, 0x41600000306ULL, 0x41600000308ULL, 0x41700000308ULL,
+ 0x41800000300ULL, 0x41800000304ULL, 0x41800000306ULL, 0x41800000308ULL, 0x41a00000301ULL,
+ 0x41e00000308ULL, 0x42300000304ULL, 0x42300000306ULL, 0x42300000308ULL, 0x4230000030bULL,
+ 0x42700000308ULL, 0x42b00000308ULL, 0x42d00000308ULL, 0x43000000306ULL, 0x43000000308ULL,
+ 0x43300000301ULL, 0x43500000300ULL, 0x43500000306ULL, 0x43500000308ULL, 0x43600000306ULL,
+ 0x43600000308ULL, 0x43700000308ULL, 0x43800000300ULL, 0x43800000304ULL, 0x43800000306ULL,
+ 0x43800000308ULL, 0x43a00000301ULL, 0x43e00000308ULL, 0x44300000304ULL, 0x44300000306ULL,
+ 0x44300000308ULL, 0x4430000030bULL, 0x44700000308ULL, 0x44b00000308ULL, 0x44d00000308ULL,
+ 0x45600000308ULL, 0x4740000030fULL, 0x4750000030fULL, 0x4d800000308ULL, 0x4d900000308ULL,
+ 0x4e800000308ULL, 0x4e900000308ULL, 0x62700000653ULL, 0x62700000654ULL, 0x62700000655ULL,
+ 0x64800000654ULL, 0x64a00000654ULL, 0x6c100000654ULL, 0x6d200000654ULL, 0x6d500000654ULL,
+ 0x9280000093cULL, 0x9300000093cULL, 0x9330000093cULL, 0x9c7000009beULL, 0x9c7000009d7ULL,
+ 0xb4700000b3eULL, 0xb4700000b56ULL, 0xb4700000b57ULL, 0xb9200000bd7ULL, 0xbc600000bbeULL,
+ 0xbc600000bd7ULL, 0xbc700000bbeULL, 0xc4600000c56ULL, 0xcbf00000cd5ULL, 0xcc600000cc2ULL,
+ 0xcc600000cd5ULL, 0xcc600000cd6ULL, 0xcca00000cd5ULL, 0xd4600000d3eULL, 0xd4600000d57ULL,
+ 0xd4700000d3eULL, 0xdd900000dcaULL, 0xdd900000dcfULL, 0xdd900000ddfULL, 0xddc00000dcaULL,
+ 0x10250000102eULL, 0x1b0500001b35ULL, 0x1b0700001b35ULL, 0x1b0900001b35ULL, 0x1b0b00001b35ULL,
+ 0x1b0d00001b35ULL, 0x1b1100001b35ULL, 0x1b3a00001b35ULL, 0x1b3c00001b35ULL, 0x1b3e00001b35ULL,
+ 0x1b3f00001b35ULL, 0x1b4200001b35ULL, 0x1e3600000304ULL, 0x1e3700000304ULL, 0x1e5a00000304ULL,
+ 0x1e5b00000304ULL, 0x1e6200000307ULL, 0x1e6300000307ULL, 0x1ea000000302ULL, 0x1ea000000306ULL,
+ 0x1ea100000302ULL, 0x1ea100000306ULL, 0x1eb800000302ULL, 0x1eb900000302ULL, 0x1ecc00000302ULL,
+ 0x1ecd00000302ULL, 0x1f0000000300ULL, 0x1f0000000301ULL, 0x1f0000000342ULL, 0x1f0000000345ULL,
+ 0x1f0100000300ULL, 0x1f0100000301ULL, 0x1f0100000342ULL, 0x1f0100000345ULL, 0x1f0200000345ULL,
+ 0x1f0300000345ULL, 0x1f0400000345ULL, 0x1f0500000345ULL, 0x1f0600000345ULL, 0x1f0700000345ULL,
+ 0x1f0800000300ULL, 0x1f0800000301ULL, 0x1f0800000342ULL, 0x1f0800000345ULL, 0x1f0900000300ULL,
+ 0x1f0900000301ULL, 0x1f0900000342ULL, 0x1f0900000345ULL, 0x1f0a00000345ULL, 0x1f0b00000345ULL,
+ 0x1f0c00000345ULL, 0x1f0d00000345ULL, 0x1f0e00000345ULL, 0x1f0f00000345ULL, 0x1f1000000300ULL,
+ 0x1f1000000301ULL, 0x1f1100000300ULL, 0x1f1100000301ULL, 0x1f1800000300ULL, 0x1f1800000301ULL,
+ 0x1f1900000300ULL, 0x1f1900000301ULL, 0x1f2000000300ULL, 0x1f2000000301ULL, 0x1f2000000342ULL,
+ 0x1f2000000345ULL, 0x1f2100000300ULL, 0x1f2100000301ULL, 0x1f2100000342ULL, 0x1f2100000345ULL,
+ 0x1f2200000345ULL, 0x1f2300000345ULL, 0x1f2400000345ULL, 0x1f2500000345ULL, 0x1f2600000345ULL,
+ 0x1f2700000345ULL, 0x1f2800000300ULL, 0x1f2800000301ULL, 0x1f2800000342ULL, 0x1f2800000345ULL,
+ 0x1f2900000300ULL, 0x1f2900000301ULL, 0x1f2900000342ULL, 0x1f2900000345ULL, 0x1f2a00000345ULL,
+ 0x1f2b00000345ULL, 0x1f2c00000345ULL, 0x1f2d00000345ULL, 0x1f2e00000345ULL, 0x1f2f00000345ULL,
+ 0x1f3000000300ULL, 0x1f3000000301ULL, 0x1f3000000342ULL, 0x1f3100000300ULL, 0x1f3100000301ULL,
+ 0x1f3100000342ULL, 0x1f3800000300ULL, 0x1f3800000301ULL, 0x1f3800000342ULL, 0x1f3900000300ULL,
+ 0x1f3900000301ULL, 0x1f3900000342ULL, 0x1f4000000300ULL, 0x1f4000000301ULL, 0x1f4100000300ULL,
+ 0x1f4100000301ULL, 0x1f4800000300ULL, 0x1f4800000301ULL, 0x1f4900000300ULL, 0x1f4900000301ULL,
+ 0x1f5000000300ULL, 0x1f5000000301ULL, 0x1f5000000342ULL, 0x1f5100000300ULL, 0x1f5100000301ULL,
+ 0x1f5100000342ULL, 0x1f5900000300ULL, 0x1f5900000301ULL, 0x1f5900000342ULL, 0x1f6000000300ULL,
+ 0x1f6000000301ULL, 0x1f6000000342ULL, 0x1f6000000345ULL, 0x1f6100000300ULL, 0x1f6100000301ULL,
+ 0x1f6100000342ULL, 0x1f6100000345ULL, 0x1f6200000345ULL, 0x1f6300000345ULL, 0x1f6400000345ULL,
+ 0x1f6500000345ULL, 0x1f6600000345ULL, 0x1f6700000345ULL, 0x1f6800000300ULL, 0x1f6800000301ULL,
+ 0x1f6800000342ULL, 0x1f6800000345ULL, 0x1f6900000300ULL, 0x1f6900000301ULL, 0x1f6900000342ULL,
+ 0x1f6900000345ULL, 0x1f6a00000345ULL, 0x1f6b00000345ULL, 0x1f6c00000345ULL, 0x1f6d00000345ULL,
+ 0x1f6e00000345ULL, 0x1f6f00000345ULL, 0x1f7000000345ULL, 0x1f7400000345ULL, 0x1f7c00000345ULL,
+ 0x1fb600000345ULL, 0x1fbf00000300ULL, 0x1fbf00000301ULL, 0x1fbf00000342ULL, 0x1fc600000345ULL,
+ 0x1ff600000345ULL, 0x1ffe00000300ULL, 0x1ffe00000301ULL, 0x1ffe00000342ULL, 0x219000000338ULL,
+ 0x219200000338ULL, 0x219400000338ULL, 0x21d000000338ULL, 0x21d200000338ULL, 0x21d400000338ULL,
+ 0x220300000338ULL, 0x220800000338ULL, 0x220b00000338ULL, 0x222300000338ULL, 0x222500000338ULL,
+ 0x223c00000338ULL, 0x224300000338ULL, 0x224500000338ULL, 0x224800000338ULL, 0x224d00000338ULL,
+ 0x226100000338ULL, 0x226400000338ULL, 0x226500000338ULL, 0x227200000338ULL, 0x227300000338ULL,
+ 0x227600000338ULL, 0x227700000338ULL, 0x227a00000338ULL, 0x227b00000338ULL, 0x227c00000338ULL,
+ 0x227d00000338ULL, 0x228200000338ULL, 0x228300000338ULL, 0x228600000338ULL, 0x228700000338ULL,
+ 0x229100000338ULL, 0x229200000338ULL, 0x22a200000338ULL, 0x22a800000338ULL, 0x22a900000338ULL,
+ 0x22ab00000338ULL, 0x22b200000338ULL, 0x22b300000338ULL, 0x22b400000338ULL, 0x22b500000338ULL,
+ 0x304600003099ULL, 0x304b00003099ULL, 0x304d00003099ULL, 0x304f00003099ULL, 0x305100003099ULL,
+ 0x305300003099ULL, 0x305500003099ULL, 0x305700003099ULL, 0x305900003099ULL, 0x305b00003099ULL,
+ 0x305d00003099ULL, 0x305f00003099ULL, 0x306100003099ULL, 0x306400003099ULL, 0x306600003099ULL,
+ 0x306800003099ULL, 0x306f00003099ULL, 0x306f0000309aULL, 0x307200003099ULL, 0x30720000309aULL,
+ 0x307500003099ULL, 0x30750000309aULL, 0x307800003099ULL, 0x30780000309aULL, 0x307b00003099ULL,
+ 0x307b0000309aULL, 0x309d00003099ULL, 0x30a600003099ULL, 0x30ab00003099ULL, 0x30ad00003099ULL,
+ 0x30af00003099ULL, 0x30b100003099ULL, 0x30b300003099ULL, 0x30b500003099ULL, 0x30b700003099ULL,
+ 0x30b900003099ULL, 0x30bb00003099ULL, 0x30bd00003099ULL, 0x30bf00003099ULL, 0x30c100003099ULL,
+ 0x30c400003099ULL, 0x30c600003099ULL, 0x30c800003099ULL, 0x30cf00003099ULL, 0x30cf0000309aULL,
+ 0x30d200003099ULL, 0x30d20000309aULL, 0x30d500003099ULL, 0x30d50000309aULL, 0x30d800003099ULL,
+ 0x30d80000309aULL, 0x30db00003099ULL, 0x30db0000309aULL, 0x30ef00003099ULL, 0x30f000003099ULL,
+ 0x30f100003099ULL, 0x30f200003099ULL, 0x30fd00003099ULL, 0x11099000110baULL,
+ 0x1109b000110baULL, 0x110a5000110baULL, 0x1113100011127ULL, 0x1113200011127ULL,
+ 0x113470001133eULL, 0x1134700011357ULL, 0x114b9000114b0ULL, 0x114b9000114baULL,
+ 0x114b9000114bdULL, 0x115b8000115afULL, 0x115b9000115afULL,
+};
+
+// the composite of k_compose_keys
+inline constexpr uint32_t k_compose_values[] = {
+ 8814, 8800, 8815, 192, 193, 194, 195, 256, 258, 550, 196, 7842, 197, 461, 512, 514, 7840, 7680,
+ 260, 7682, 7684, 7686, 262, 264, 266, 268, 199, 7690, 270, 7692, 7696, 7698, 7694, 200, 201,
+ 202, 7868, 274, 276, 278, 203, 7866, 282, 516, 518, 7864, 552, 280, 7704, 7706, 7710, 500, 284,
+ 7712, 286, 288, 486, 290, 292, 7714, 7718, 542, 7716, 7720, 7722, 204, 205, 206, 296, 298, 300,
+ 304, 207, 7880, 463, 520, 522, 7882, 302, 7724, 308, 7728, 488, 7730, 310, 7732, 313, 317,
+ 7734, 315, 7740, 7738, 7742, 7744, 7746, 504, 323, 209, 7748, 327, 7750, 325, 7754, 7752, 210,
+ 211, 212, 213, 332, 334, 558, 214, 7886, 336, 465, 524, 526, 416, 7884, 490, 7764, 7766, 340,
+ 7768, 344, 528, 530, 7770, 342, 7774, 346, 348, 7776, 352, 7778, 536, 350, 7786, 356, 7788,
+ 538, 354, 7792, 7790, 217, 218, 219, 360, 362, 364, 220, 7910, 366, 368, 467, 532, 534, 431,
+ 7908, 7794, 370, 7798, 7796, 7804, 7806, 7808, 7810, 372, 7814, 7812, 7816, 7818, 7820, 7922,
+ 221, 374, 7928, 562, 7822, 376, 7926, 7924, 377, 7824, 379, 381, 7826, 7828, 224, 225, 226,
+ 227, 257, 259, 551, 228, 7843, 229, 462, 513, 515, 7841, 7681, 261, 7683, 7685, 7687, 263, 265,
+ 267, 269, 231, 7691, 271, 7693, 7697, 7699, 7695, 232, 233, 234, 7869, 275, 277, 279, 235,
+ 7867, 283, 517, 519, 7865, 553, 281, 7705, 7707, 7711, 501, 285, 7713, 287, 289, 487, 291, 293,
+ 7715, 7719, 543, 7717, 7721, 7723, 7830, 236, 237, 238, 297, 299, 301, 239, 7881, 464, 521,
+ 523, 7883, 303, 7725, 309, 496, 7729, 489, 7731, 311, 7733, 314, 318, 7735, 316, 7741, 7739,
+ 7743, 7745, 7747, 505, 324, 241, 7749, 328, 7751, 326, 7755, 7753, 242, 243, 244, 245, 333,
+ 335, 559, 246, 7887, 337, 466, 525, 527, 417, 7885, 491, 7765, 7767, 341, 7769, 345, 529, 531,
+ 7771, 343, 7775, 347, 349, 7777, 353, 7779, 537, 351, 7787, 7831, 357, 7789, 539, 355, 7793,
+ 7791, 249, 250, 251, 361, 363, 365, 252, 7911, 367, 369, 468, 533, 535, 432, 7909, 7795, 371,
+ 7799, 7797, 7805, 7807, 7809, 7811, 373, 7815, 7813, 7832, 7817, 7819, 7821, 7923, 253, 375,
+ 7929, 563, 7823, 255, 7927, 7833, 7925, 378, 7825, 380, 382, 7827, 7829, 8173, 901, 8129, 7846,
+ 7844, 7850, 7848, 478, 506, 508, 482, 7688, 7872, 7870, 7876, 7874, 7726, 7890, 7888, 7894,
+ 7892, 7756, 556, 7758, 554, 510, 475, 471, 469, 473, 7847, 7845, 7851, 7849, 479, 507, 509,
+ 483, 7689, 7873, 7871, 7877, 7875, 7727, 7891, 7889, 7895, 7893, 7757, 557, 7759, 555, 511,
+ 476, 472, 470, 474, 7856, 7854, 7860, 7858, 7857, 7855, 7861, 7859, 7700, 7702, 7701, 7703,
+ 7760, 7762, 7761, 7763, 7780, 7781, 7782, 7783, 7800, 7801, 7802, 7803, 7835, 7900, 7898, 7904,
+ 7902, 7906, 7901, 7899, 7905, 7903, 7907, 7914, 7912, 7918, 7916, 7920, 7915, 7913, 7919, 7917,
+ 7921, 494, 492, 493, 480, 481, 7708, 7709, 560, 561, 495, 8122, 902, 8121, 8120, 7944, 7945,
+ 8124, 8136, 904, 7960, 7961, 8138, 905, 7976, 7977, 8140, 8154, 906, 8153, 8152, 938, 7992,
+ 7993, 8184, 908, 8008, 8009, 8172, 8170, 910, 8169, 8168, 939, 8025, 8186, 911, 8040, 8041,
+ 8188, 8116, 8132, 8048, 940, 8113, 8112, 7936, 7937, 8118, 8115, 8050, 941, 7952, 7953, 8052,
+ 942, 7968, 7969, 8134, 8131, 8054, 943, 8145, 8144, 970, 7984, 7985, 8150, 8056, 972, 8000,
+ 8001, 8164, 8165, 8058, 973, 8161, 8160, 971, 8016, 8017, 8166, 8060, 974, 8032, 8033, 8182,
+ 8179, 8146, 912, 8151, 8162, 944, 8167, 8180, 979, 980, 1031, 1232, 1234, 1027, 1024, 1238,
+ 1025, 1217, 1244, 1246, 1037, 1250, 1049, 1252, 1036, 1254, 1262, 1038, 1264, 1266, 1268, 1272,
+ 1260, 1233, 1235, 1107, 1104, 1239, 1105, 1218, 1245, 1247, 1117, 1251, 1081, 1253, 1116, 1255,
+ 1263, 1118, 1265, 1267, 1269, 1273, 1261, 1111, 1142, 1143, 1242, 1243, 1258, 1259, 1570, 1571,
+ 1573, 1572, 1574, 1730, 1747, 1728, 2345, 2353, 2356, 2507, 2508, 2891, 2888, 2892, 2964, 3018,
+ 3020, 3019, 3144, 3264, 3274, 3271, 3272, 3275, 3402, 3404, 3403, 3546, 3548, 3550, 3549, 4134,
+ 6918, 6920, 6922, 6924, 6926, 6930, 6971, 6973, 6976, 6977, 6979, 7736, 7737, 7772, 7773, 7784,
+ 7785, 7852, 7862, 7853, 7863, 7878, 7879, 7896, 7897, 7938, 7940, 7942, 8064, 7939, 7941, 7943,
+ 8065, 8066, 8067, 8068, 8069, 8070, 8071, 7946, 7948, 7950, 8072, 7947, 7949, 7951, 8073, 8074,
+ 8075, 8076, 8077, 8078, 8079, 7954, 7956, 7955, 7957, 7962, 7964, 7963, 7965, 7970, 7972, 7974,
+ 8080, 7971, 7973, 7975, 8081, 8082, 8083, 8084, 8085, 8086, 8087, 7978, 7980, 7982, 8088, 7979,
+ 7981, 7983, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 7986, 7988, 7990, 7987, 7989, 7991, 7994,
+ 7996, 7998, 7995, 7997, 7999, 8002, 8004, 8003, 8005, 8010, 8012, 8011, 8013, 8018, 8020, 8022,
+ 8019, 8021, 8023, 8027, 8029, 8031, 8034, 8036, 8038, 8096, 8035, 8037, 8039, 8097, 8098, 8099,
+ 8100, 8101, 8102, 8103, 8042, 8044, 8046, 8104, 8043, 8045, 8047, 8105, 8106, 8107, 8108, 8109,
+ 8110, 8111, 8114, 8130, 8178, 8119, 8141, 8142, 8143, 8135, 8183, 8157, 8158, 8159, 8602, 8603,
+ 8622, 8653, 8655, 8654, 8708, 8713, 8716, 8740, 8742, 8769, 8772, 8775, 8777, 8813, 8802, 8816,
+ 8817, 8820, 8821, 8824, 8825, 8832, 8833, 8928, 8929, 8836, 8837, 8840, 8841, 8930, 8931, 8876,
+ 8877, 8878, 8879, 8938, 8939, 8940, 8941, 12436, 12364, 12366, 12368, 12370, 12372, 12374,
+ 12376, 12378, 12380, 12382, 12384, 12386, 12389, 12391, 12393, 12400, 12401, 12403, 12404,
+ 12406, 12407, 12409, 12410, 12412, 12413, 12446, 12532, 12460, 12462, 12464, 12466, 12468,
+ 12470, 12472, 12474, 12476, 12478, 12480, 12482, 12485, 12487, 12489, 12496, 12497, 12499,
+ 12500, 12502, 12503, 12505, 12506, 12508, 12509, 12535, 12536, 12537, 12538, 12542, 69786,
+ 69788, 69803, 69934, 69935, 70475, 70476, 70844, 70843, 70846, 71098, 71099,
+};
diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cpp
new file mode 100644
index 00000000..d29ce37e
--- /dev/null
+++ b/laya-cpp/tests/tokenizer_test.cpp
@@ -0,0 +1,636 @@
+// laya::tokenizer against the `tokenizers` crate (through the Python-recorded goldens) and against
+// hand-built tokenizers. Ported from laya-go/tokenizer's tests.
+#include <cstdlib>
+#include <fstream>
+#include <mutex>
+#include <thread>
+
+#include "golden.hpp"
+#include "laya/pyvalue.hpp"
+#include "laya/sha256.hpp"
+#include "laya/tokenizer.hpp"
+#include "laya/unicodedata.hpp"
+
+using laya::tokenizer::Form;
+using laya::tokenizer::Tokenizer;
+using laya_test::At;
+using laya_test::Value;
+namespace tk = laya::tokenizer;
+namespace ud = laya::unicodedata;
+namespace pv = laya::pyvalue;
+
+namespace {
+
+const char* const kShipped[] = {"english", "multilingual"};
+
+std::vector<int32_t> Ints(const Value& v) {
+ std::vector<int32_t> out;
+ for (const Value& x : v.items()) out.push_back(int32_t(x.as_int()));
+ return out;
+}
+std::string Join(const std::vector<std::string>& v) {
+ std::string out;
+ for (const auto& s : v) out += s + "\x01";
+ return out;
+}
+std::string Ids(const std::vector<int32_t>& v) {
+ std::string out = "[";
+ for (size_t i = 0; i < v.size() && i < 24; i++) out += std::to_string(v[i]) + " ";
+ return out + (v.size() > 24 ? "... " : "") + "(" + std::to_string(v.size()) + ")]";
+}
+std::string Hex(std::string_view s) {
+ std::string out;
+ for (unsigned char c : s) {
+ char b[4];
+ std::snprintf(b, sizeof b, "%02x", c);
+ out += b;
+ }
+ return out;
+}
+
+// A shipped checkpoint's tokenizer, loaded once. Without LAYA_CHECKPOINTS the test is skipped
+// locally and FAILS under LAYA_REQUIRE_GOLDENS=1 (CI), so a missing checkpoint cannot pass as green.
+const Tokenizer* Checkpoint(const std::string& name) {
+ static std::mutex mu;
+ static std::map<std::string, Tokenizer> loaded;
+ std::lock_guard<std::mutex> lock(mu);
+ auto it = loaded.find(name);
+ if (it != loaded.end()) return &it->second;
+ const char* root = std::getenv("LAYA_CHECKPOINTS");
+ if (root == nullptr || *root == 0) return nullptr;
+ auto t = Tokenizer::LoadDir(std::string(root) + "/" + name);
+ if (!t.ok()) {
+ ADD_FAILURE() << "load the " << name << " tokenizer: " << t.error();
+ return nullptr;
+ }
+ return &loaded.emplace(name, std::move(t).value()).first->second;
+}
+
+#define LAYA_CHECKPOINT(var, name) \
+ const Tokenizer* var = Checkpoint(name); \
+ if (var == nullptr) { \
+ const char* req = std::getenv("LAYA_REQUIRE_GOLDENS"); \
+ if (req != nullptr && std::string(req) == "1" && !HasFailure()) { \
+ ADD_FAILURE() << "LAYA_CHECKPOINTS is unset but LAYA_REQUIRE_GOLDENS=1"; \
+ } else if (!HasFailure()) { \
+ GTEST_SKIP() << "LAYA_CHECKPOINTS unset (local run)"; \
+ } \
+ return; \
+ }
+
+Tokenizer MustParse(const std::string& json, const char* config = nullptr) {
+ auto t = Tokenizer::Parse(json, config == nullptr ? std::nullopt : std::optional<std::string_view>(config));
+ EXPECT_TRUE(t.ok()) << (t.ok() ? "" : t.error());
+ if (!t.ok()) std::abort();
+ return std::move(t).value();
+}
+
+// A tiny byte-level tokenizer built in the test, so the loader is checked without a checkpoint.
+std::string ByteLevelSpec(const std::string& extra) {
+ std::string vocab;
+ for (int b = 0, lifted = 0; b < 256; b++) {
+ char32_t r;
+ if ((b >= '!' && b <= '~') || (b >= 0xA1 && b <= 0xAC) || (b >= 0xAE && b <= 0xFF)) r = char32_t(b);
+ else r = char32_t(256 + lifted++);
+ std::string key;
+ ud::AppendRune(key, r);
+ if (r == '"' || r == '\\') key = "\\" + key;
+ vocab += "\"" + key + "\":" + std::to_string(b) + ",";
+ }
+ vocab += "\"ab\":256";
+ return "{\"model\":{\"type\":\"BPE\",\"vocab\":{" + vocab + "},\"merges\":[\"a b\"]},\"normalizer\":null," +
+ "\"pre_tokenizer\":{\"type\":\"ByteLevel\",\"add_prefix_space\":false,\"use_regex\":true}" + extra + "}";
+}
+std::string Replace(std::string s, const std::string& from, const std::string& to) {
+ size_t at = s.find(from);
+ if (at != std::string::npos) s.replace(at, from.size(), to);
+ return s;
+}
+
+std::string Digest(const std::function<void(const std::function<void(const std::string&)>&)>& rows) {
+ laya::Sha256 h;
+ rows([&](const std::string& s) { h.Update(s); });
+ return laya::ToHex(h.Finish());
+}
+std::string Mapped(char32_t cp, const std::string& text) {
+ std::string out = std::to_string(uint32_t(cp)) + ":";
+ bool first = true;
+ std::string_view rest(text);
+ while (!rest.empty()) {
+ auto d = ud::DecodeRune(rest);
+ out += (first ? "" : ",") + std::to_string(uint32_t(d.rune));
+ first = false;
+ rest.remove_prefix(d.width);
+ }
+ return out + ";";
+}
+std::string One(char32_t cp) {
+ std::string s;
+ ud::AppendRune(s, cp);
+ return s;
+}
+const char* PyBool(bool b) { return b ? "True" : "False"; }
+
+} // namespace
+
+// ---- normalization, lowercase and the single_word flank over every code point ----------------------
+class FormSweep : public ::testing::TestWithParam<std::pair<const char*, Form>> {};
+TEST_P(FormSweep, MatchesTheCrateOverAllUnicode) {
+ LAYA_GOLDEN_LIST(sweep, "go_extra.json", "normalize", "sweep");
+ Form form = GetParam().second;
+ std::string got = Digest([&](const auto& yield) {
+ for (char32_t r = 0; r < 0x110000; r++) {
+ if (r >= 0xD800 && r <= 0xDFFF) continue;
+ yield(Mapped(r, tk::Normalize(One(r), form)));
+ }
+ });
+ EXPECT_EQ(got, sweep->Find(GetParam().first)->as_string()) << "some single code point differs from the crate";
+}
+INSTANTIATE_TEST_SUITE_P(Golden, FormSweep,
+ ::testing::Values(std::make_pair("NFC", Form::NFC), std::make_pair("NFD", Form::NFD),
+ std::make_pair("NFKC", Form::NFKC), std::make_pair("NFKD", Form::NFKD)));
+
+TEST(TokenizerNormalize, RandomStringsMatchTheCrate) {
+ LAYA_GOLDEN_LIST(cases, "go_extra.json", "normalize", "cases");
+ ASSERT_GE(cases->size(), 1000u);
+ for (size_t i = 0; i < cases->size(); i++) {
+ const Value& c = cases->value(i);
+ const std::string& text = c.Find("text")->as_string();
+ const std::pair<const char*, Form> forms[] = {{"nfc", Form::NFC}, {"nfd", Form::NFD}, {"nfkc", Form::NFKC}, {"nfkd", Form::NFKD}};
+ for (const auto& f : forms) {
+ ASSERT_EQ(tk::Normalize(text, f.second), c.Find(f.first)->as_string()) << "case " << i << " " << f.first << " of " << Hex(text);
+ }
+ }
+}
+
+TEST(TokenizerNormalize, LowercaseMatchesTheCrateOverAllUnicode) {
+ LAYA_GOLDEN_LIST(want, "go_extra.json", "lowercase", "sweep");
+ std::string got = Digest([&](const auto& yield) {
+ for (char32_t r = 0; r < 0x110000; r++) {
+ if (r >= 0xD800 && r <= 0xDFFF) continue;
+ yield(Mapped(r, tk::Lowercase(One(r))));
+ }
+ });
+ EXPECT_EQ(got, want->as_string());
+ EXPECT_EQ(tk::Lowercase("\xCE\xA3"), "\xCF\x83") << "no final-sigma rule: the crate lowercases per character";
+}
+
+// A single_word added token is refused beside a word character. Which characters those are is the
+// crate's rule, not an alphanumeric test: it counts combining marks and joiners and not other numbers.
+TEST(TokenizerAdded, SingleWordFlankMatchesTheCrate) {
+ LAYA_GOLDEN_LIST(sw, "go_extra.json", "single_word");
+ Tokenizer tok = MustParse(
+ R"({"model":{"type":"BPE","vocab":{"a":0,"<unk>":2},"merges":[],"unk_token":"<unk>"},
+ "added_tokens":[{"content":"TOK","id":1,"single_word":true,"special":true}]})");
+ auto standalone = [&](const std::string& s) {
+ for (int32_t id : tok.Encode(s)) {
+ if (id == 1) return std::string("1");
+ }
+ return std::string("0");
+ };
+ for (const char* side : {"left_standalone", "right_standalone"}) {
+ bool left = std::string(side) == "left_standalone";
+ std::string got = Digest([&](const auto& yield) {
+ for (char32_t r = 0; r < 0x110000; r++) {
+ if (r >= 0xD800 && r <= 0xDFFF) {
+ yield("0");
+ continue;
+ }
+ yield(standalone(left ? One(r) + "TOK" : "TOK" + One(r)));
+ }
+ });
+ EXPECT_EQ(got, sw->Find(side)->as_string()) << side;
+ }
+}
+
+// ---- pre-tokenizers and normalizer specs ---------------------------------------------------------------
+TEST(TokenizerPre, PreTokenizersMatchTheCrate) {
+ LAYA_GOLDEN_LIST(pre, "go_extra.json", "pretokenize");
+ const Value& texts = *pre->Find("texts");
+ size_t checked = 0;
+ auto check = [&](const std::string& name, const std::string& spec, const Value& want) {
+ ASSERT_EQ(want.size(), texts.size()) << name;
+ for (size_t i = 0; i < texts.size(); i++) {
+ auto got = tk::RunPreTokenizerSpec(spec, texts.value(i).as_string());
+ ASSERT_TRUE(got.ok()) << got.error();
+ std::vector<std::string> w;
+ for (const Value& x : want.value(i).items()) w.push_back(x.as_string());
+ ASSERT_EQ(Join(got.value()), Join(w)) << name << " of " << Hex(texts.value(i).as_string());
+ checked++;
+ }
+ };
+ for (bool prefix : {false, true}) {
+ for (bool regex : {true, false}) {
+ std::string key = std::string("add_prefix_space=") + PyBool(prefix) + ",use_regex=" + PyBool(regex);
+ check("ByteLevel/" + key,
+ std::string("{\"type\":\"ByteLevel\",\"add_prefix_space\":") + (prefix ? "true" : "false") + ",\"use_regex\":" + (regex ? "true" : "false") + "}",
+ *At(pre, {"byte_level", key.c_str()}));
+ }
+ }
+ for (const char* scheme : {"always", "first", "never"}) {
+ for (bool split : {true, false}) {
+ std::string key = std::string(scheme) + ",split=" + PyBool(split);
+ check("Metaspace/" + key,
+ std::string("{\"type\":\"Metaspace\",\"replacement\":\"\xE2\x96\x81\",\"prepend_scheme\":\"") + scheme + "\",\"split\":" + (split ? "true" : "false") + "}",
+ *At(pre, {"metaspace", key.c_str()}));
+ }
+ }
+ EXPECT_GE(checked, 10u * 1000u);
+}
+
+TEST(TokenizerNormalize, NormalizerSpecsMatchTheCrate) {
+ LAYA_GOLDEN_LIST(fam, "go_extra.json", "normalizers");
+ const Value& inputs = *fam->Find("inputs");
+ const Value& specs = *fam->Find("specs");
+ ASSERT_GE(specs.size(), 12u);
+ for (size_t s = 0; s < specs.size(); s++) {
+ auto spec = pv::Dumps(*specs.value(s).Find("spec"));
+ ASSERT_TRUE(spec.ok());
+ const Value& outs = *specs.value(s).Find("outputs");
+ for (size_t j = 0; j < inputs.size(); j++) {
+ auto got = tk::RunNormalizerSpec(spec.value(), inputs.value(j).as_string());
+ ASSERT_TRUE(got.ok()) << spec.value() << ": " << got.error();
+ ASSERT_EQ(got.value(), outs.value(j).as_string()) << spec.value() << " on " << Hex(inputs.value(j).as_string());
+ }
+ }
+}
+
+// ---- the shipped tokenizers against Python ------------------------------------------------------------
+class Shipped : public ::testing::TestWithParam<const char*> {};
+INSTANTIATE_TEST_SUITE_P(Checkpoint, Shipped, ::testing::ValuesIn(kShipped));
+
+TEST_P(Shipped, CorpusIdsIdentityAndSpecialsMatchPython) {
+ std::string name = GetParam();
+ LAYA_GOLDEN_LIST(side, "tokenizer_ids.json", name.c_str());
+ LAYA_CHECKPOINT(tok, name);
+ const Value* corpus_text = At(laya_test::Load("tokenizer_ids.json", true), {"corpus_text"});
+ ASSERT_NE(corpus_text, nullptr);
+ const Value& id = *side->Find("identity");
+ size_t want_vocab = size_t(id.Find("vocab_size")->as_int());
+ if (const Value* extra = id.Find("added_tokens_outside_vocab"); extra != nullptr && !extra->is_null()) want_vocab += size_t(extra->as_int());
+ EXPECT_EQ(tok->VocabSize(), want_vocab);
+ const Value& sp = *side->Find("special_ids");
+ EXPECT_EQ(tok->Cls(), std::optional<int32_t>(int32_t(sp.Find("cls")->as_int())));
+ EXPECT_EQ(tok->Sep(), std::optional<int32_t>(int32_t(sp.Find("sep")->as_int())));
+ EXPECT_EQ(tok->Mask(), std::optional<int32_t>(int32_t(sp.Find("mask")->as_int())));
+ EXPECT_EQ(tok->Pad(), std::optional<int32_t>(int32_t(sp.Find("pad")->as_int())));
+ EXPECT_EQ(tok->MaskToken(), sp.Find("mask_token")->as_string());
+ const Value& corpus = *side->Find("corpus");
+ ASSERT_GE(corpus.size(), 35u);
+ for (size_t i = 0; i < corpus.size(); i++) {
+ const std::string& text = corpus_text->Find(corpus.key(i))->as_string();
+ auto got = tok->Encode(text);
+ EXPECT_EQ(got, Ints(corpus.value(i))) << corpus.key(i) << " " << Ids(got);
+ }
+ const Value& trunc = *side->Find("option_truncation");
+ std::string text = " " + trunc.Find("text")->as_string();
+ EXPECT_EQ(tok->Encode(text), Ints(*trunc.Find("uncapped")));
+ EXPECT_EQ(tok->Encode(text, 48), Ints(*trunc.Find("capped_48")));
+}
+
+TEST_P(Shipped, DecodeMatchesPythonWholeAndSliced) {
+ std::string name = GetParam();
+ LAYA_GOLDEN_LIST(entry, "decode_text.json", "by", name.c_str());
+ LAYA_CHECKPOINT(tok, name);
+ const Value& cases = *entry->Find("cases");
+ ASSERT_GE(cases.size(), 23u);
+ size_t slices = 0;
+ for (const Value& c : cases.items()) {
+ const std::string& text = c.Find("text")->as_string();
+ auto ids = Ints(*c.Find("ids"));
+ EXPECT_EQ(tok->Encode(text), ids) << Hex(text);
+ EXPECT_EQ(tok->Decode(ids, true), c.Find("decoded")->as_string());
+ for (const Value& s : c.Find("slices")->items()) {
+ slices++;
+ size_t start = size_t(s.Find("start")->as_int());
+ size_t end = std::min(start + size_t(s.Find("size")->as_int()), ids.size());
+ std::vector<int32_t> part(ids.begin() + long(start), ids.begin() + long(end));
+ EXPECT_EQ(tok->Decode(part, true), s.Find("decoded")->as_string()) << "slice " << start;
+ }
+ if (c.Find("slices")->size() == 0) EXPECT_TRUE(text.empty() && ids.empty());
+ const Value& w = *c.Find("with_specials");
+ auto with = Ints(*w.Find("ids"));
+ EXPECT_EQ(tok->Decode(with, true), w.Find("skipping_specials")->as_string());
+ EXPECT_EQ(tok->Decode(with, false), w.Find("kept")->as_string());
+ }
+ EXPECT_GE(slices, 150u) << "the slice cases are the point of this family";
+}
+
+TEST_P(Shipped, EncodeFuzzMatchesTheCrate) {
+ std::string name = GetParam();
+ LAYA_GOLDEN_LIST(side, "go_extra.json", "encode_fuzz", name.c_str());
+ LAYA_CHECKPOINT(tok, name);
+ const Value& texts = *side->Find("texts");
+ const Value& ids = *side->Find("ids");
+ ASSERT_GE(texts.size(), 500u);
+ int bad = 0;
+ for (size_t i = 0; i < texts.size() && bad < 5; i++) {
+ auto got = tok->Encode(texts.value(i).as_string());
+ if (got != Ints(ids.value(i))) {
+ ADD_FAILURE() << "Encode(" << Hex(texts.value(i).as_string()) << ") = " << Ids(got) << ", want " << Ids(Ints(ids.value(i)));
+ bad++;
+ }
+ }
+}
+
+TEST_P(Shipped, DecodeFuzzMatchesTheCrate) {
+ std::string name = GetParam();
+ LAYA_GOLDEN_LIST(cases, "go_extra.json", "decode_fuzz", name.c_str(), "cases");
+ LAYA_CHECKPOINT(tok, name);
+ int bad = 0;
+ for (const Value& c : cases->items()) {
+ auto ids = Ints(*c.Find("ids"));
+ for (bool skip : {true, false}) {
+ if (tok->Decode(ids, skip) != c.Find(skip ? "skipping" : "keeping")->as_string() && bad++ < 5) {
+ ADD_FAILURE() << "Decode(" << Ids(ids) << ", skip=" << skip << ")";
+ }
+ }
+ }
+}
+
+// A Python str is always valid, so invalid bytes reach the tokenizer as bytes.decode('utf-8','replace').
+TEST_P(Shipped, InvalidUtf8EncodesLikePython) {
+ std::string name = GetParam();
+ LAYA_GOLDEN_LIST(fam, "go_extra.json", "invalid_utf8");
+ LAYA_CHECKPOINT(tok, name);
+ const Value& hexes = *fam->Find("hex");
+ const Value& want = *At(fam, {"ids", name.c_str()});
+ ASSERT_GE(want.size(), 600u);
+ for (size_t i = 0; i < want.size(); i++) {
+ const std::string& h = hexes.value(i).as_string();
+ std::string raw;
+ for (size_t k = 0; k + 1 < h.size(); k += 2) raw.push_back(char(std::stoi(h.substr(k, 2), nullptr, 16)));
+ ASSERT_EQ(tok->Encode(raw), Ints(want.value(i))) << h;
+ }
+}
+
+// A loaded Tokenizer is read-only, so any number of threads may share it.
+TEST_P(Shipped, ConcurrentUse) {
+ std::string name = GetParam();
+ LAYA_GOLDEN_LIST(side, "go_extra.json", "encode_fuzz", name.c_str());
+ LAYA_CHECKPOINT(tok, name);
+ const Value& texts = *side->Find("texts");
+ const Value& ids = *side->Find("ids");
+ std::vector<std::string> errors(8);
+ std::vector<std::thread> threads;
+ for (int g = 0; g < 8; g++) {
+ threads.emplace_back([&, g] {
+ for (size_t i = size_t(g); i < texts.size(); i += size_t(1 + g % 3)) {
+ auto got = tok->Encode(texts.value(i).as_string());
+ if (got != Ints(ids.value(i))) {
+ errors[size_t(g)] = "thread " + std::to_string(g) + " text " + std::to_string(i);
+ return;
+ }
+ (void)tok->Decode(got, true);
+ }
+ });
+ }
+ for (auto& t : threads) t.join();
+ for (const auto& e : errors) EXPECT_EQ(e, "");
+}
+
+TEST(TokenizerDecode, DecoderChainsAndEdges) {
+ LAYA_GOLDEN_LIST(by, "decode_text.json", "by");
+ EXPECT_EQ(At(by, {"english", "decoder", "type"})->as_string(), "ByteLevel");
+ std::string types;
+ for (const Value& s : At(by, {"multilingual", "decoder", "decoders"})->items()) types += s.Find("type")->as_string() + ",";
+ EXPECT_EQ(types, "Replace,ByteFallback,Fuse,");
+}
+
+TEST(TokenizerDecode, SyntheticDecoderShapesAndMixedToken) {
+ LAYA_GOLDEN_LIST(shapes, "decode_text.json", "synthetic_decoders");
+ ASSERT_GE(shapes->size(), 5u);
+ for (size_t i = 0; i < shapes->size(); i++) {
+ const Value& shape = shapes->value(i);
+ auto spec = pv::Dumps(*shape.Find("spec"));
+ ASSERT_TRUE(spec.ok());
+ Tokenizer tok = MustParse(spec.value());
+ ASSERT_GT(shape.Find("cases")->size(), 0u);
+ for (const Value& c : shape.Find("cases")->items()) {
+ EXPECT_EQ(tok.Decode(Ints(*c.Find("ids")), false), c.Find("decoded")->as_string()) << shapes->key(i);
+ }
+ if (const Value* panics = shape.Find("crate_panics")) { // the crate panics on these; this port returns empty text
+ for (const Value& p : panics->items()) EXPECT_EQ(tok.Decode(Ints(p), false), "") << shapes->key(i);
+ }
+ }
+ const Value* mixed = At(laya_test::Load("decode_text.json", true), {"mixed_token"});
+ ASSERT_NE(mixed, nullptr);
+ EXPECT_EQ(mixed->Find("scope")->as_string(), "token");
+ auto spec = pv::Dumps(*mixed->Find("spec"));
+ ASSERT_TRUE(spec.ok());
+ Tokenizer tok = MustParse(spec.value());
+ for (const Value& c : mixed->Find("cases")->items()) {
+ EXPECT_EQ(tok.Decode(Ints(*c.Find("ids")), false), c.Find("decoded")->as_string()) << c.Find("why")->as_string();
+ }
+}
+
+TEST(TokenizerDecode, EdgesOnTheEnglishAndMultilingualCheckpoints) {
+ LAYA_CHECKPOINT(en, "english");
+ LAYA_CHECKPOINT(ml, "multilingual");
+ EXPECT_EQ(en->Decode({}, true), "");
+ EXPECT_EQ(en->Decode({1 << 30}, true), "");
+ int32_t a = en->Encode("a")[0];
+ EXPECT_EQ(en->Decode({1 << 30, a, -1}, true), "a");
+ EXPECT_EQ(en->Decode({50275}, true), " ") << "the literal-whitespace added token survives decoding";
+ for (const char* s : {" ", " "}) EXPECT_EQ(en->Decode(en->Encode(s), true), s);
+ EXPECT_EQ(ml->Decode(ml->Encode("Hello world"), true), " Hello world");
+ auto ids = ml->Encode("\xF3\xB0\x80\x80"); // U+F0000, four bytes of byte fallback
+ ASSERT_GE(ids.size(), 3u);
+ EXPECT_EQ(ml->Decode({ids[0], ids[1], ids[2]}, true), " \xEF\xBF\xBD\xEF\xBF\xBD");
+ EXPECT_EQ(ml->Decode(ids, true), " \xF3\xB0\x80\x80");
+ auto full = en->Encode("hello world, this is a longer sentence");
+ EXPECT_TRUE(en->Encode("hello world", 0).empty());
+ for (size_t n = 1; n <= full.size(); n++) {
+ EXPECT_EQ(en->Encode("hello world, this is a longer sentence", int64_t(n)), std::vector<int32_t>(full.begin(), full.begin() + long(n)));
+ }
+ EXPECT_EQ(en->Encode("hello", 100), en->Encode("hello"));
+}
+
+// ---- loader behaviour on hand-built tokenizers ------------------------------------------------------
+TEST(TokenizerLoader, SmallTokenizerRoundTrips) {
+ Tokenizer tok = MustParse(ByteLevelSpec(",\"decoder\":{\"type\":\"ByteLevel\"}"));
+ auto ids = tok.Encode("ab ab");
+ EXPECT_EQ(tok.Decode(ids, true), "ab ab");
+ EXPECT_EQ(ids[0], 256);
+ EXPECT_EQ(tok.VocabSize(), 257u);
+ EXPECT_EQ(tok.HighestId(), 256);
+ EXPECT_EQ(tok.IdToToken(256), std::optional<std::string>("ab"));
+ EXPECT_FALSE(tok.IdToToken(-1).has_value());
+ EXPECT_EQ(tok.TokenToId("ab"), std::optional<int32_t>(256));
+}
+
+TEST(TokenizerLoader, Refuses) {
+ std::string good = ByteLevelSpec("");
+ struct Case {
+ const char* name;
+ std::string spec;
+ bool unsupported;
+ } cases[] = {
+ {"a non-BPE model", Replace(good, "\"BPE\"", "\"WordPiece\""), true},
+ {"an unknown normalizer", Replace(good, "\"normalizer\":null", "\"normalizer\":{\"type\":\"BertNormalizer\"}"), true},
+ {"an unknown pre-tokenizer", Replace(good, "\"pre_tokenizer\":{\"type\":\"ByteLevel\"", "\"pre_tokenizer\":{\"type\":\"Whitespace\""), true},
+ {"an unknown decoder", ByteLevelSpec(",\"decoder\":{\"type\":\"WordPiece\"}"), true},
+ {"a merge naming a symbol the vocabulary lacks", Replace(good, "\"a b\"", "\"a zz\""), false},
+ {"a merge whose result is missing", Replace(good, "\"a b\"", "\"a c\""), false},
+ {"a merge that is not a pair", Replace(good, "\"a b\"", "[\"a\"]"), false},
+ {"a space-joined merge with an empty half", Replace(good, "\"a b\"", "\"a \""), false},
+ {"an added token with no id", ByteLevelSpec(",\"added_tokens\":[{\"content\":\"x\"}]"), false},
+ {"two added tokens sharing an id", ByteLevelSpec(",\"added_tokens\":[{\"content\":\"x\",\"id\":300},{\"content\":\"y\",\"id\":300}]"), false},
+ {"a vocabulary giving one id to two tokens", Replace(good, "\"ab\":256", "\"ab\":0"), false},
+ {"no vocabulary", "{\"model\":{\"type\":\"BPE\",\"merges\":[]}}", false},
+ {"not JSON", "{", false},
+ {"a lossy non-byte-level tokenizer", "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0},\"merges\":[]}}", true},
+ };
+ for (const auto& c : cases) {
+ auto t = Tokenizer::Parse(c.spec);
+ ASSERT_FALSE(t.ok()) << c.name << " must be refused, not loaded";
+ EXPECT_EQ(tk::IsUnsupported(t.error()), c.unsupported) << c.name << ": " << t.error();
+ }
+ auto partial = Tokenizer::Parse("{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0},\"merges\":[]},\"pre_tokenizer\":{\"type\":\"ByteLevel\"}}");
+ ASSERT_FALSE(partial.ok());
+ EXPECT_NE(partial.error().find("absent from its vocabulary"), std::string::npos) << partial.error();
+}
+
+TEST(TokenizerLoader, UnknownTokenAndFuseUnk) {
+ std::string with_unk = "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"<unk>\":1},\"merges\":[],\"unk_token\":\"<unk>\"}}";
+ struct Case {
+ bool fuse;
+ const char* text;
+ std::vector<int32_t> want;
+ } cases[] = {{false, "azz a", {0, 1, 1, 1, 0}}, {false, "zzz", {1, 1, 1}}, {false, "zz a zz", {1, 1, 1, 0, 1, 1, 1}},
+ {true, "azz a", {0, 1, 0}}, {true, "zzz", {1}}, {true, "zz a zz", {1, 0, 1}}};
+ for (const auto& c : cases) {
+ Tokenizer tok = MustParse(c.fuse ? Replace(with_unk, "\"unk_token\"", "\"fuse_unk\":true,\"unk_token\"") : with_unk);
+ EXPECT_EQ(tok.Encode(c.text), c.want) << "fuse=" << c.fuse << " " << c.text;
+ }
+}
+
+TEST(TokenizerLoader, IdsFarBeyondTheVocabularyAreRefused) {
+ for (const char* id : {"2147483647", "1000000000", "70000"}) {
+ auto t = Tokenizer::Parse(std::string("{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"<unk>\":") + id + "},\"merges\":[],\"unk_token\":\"<unk>\"}}");
+ ASSERT_FALSE(t.ok()) << id;
+ EXPECT_NE(t.error().find("sparse"), std::string::npos) << t.error();
+ }
+ EXPECT_TRUE(Tokenizer::Parse("{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"<unk>\":9000},\"merges\":[],\"unk_token\":\"<unk>\"}}").ok());
+}
+
+TEST(TokenizerAdded, LongestLstripRstripSingleWordNormalized) {
+ Tokenizer tok = MustParse(ByteLevelSpec(
+ ",\"added_tokens\":[{\"content\":\"<S>\",\"id\":300,\"special\":true,\"normalized\":false},"
+ "{\"content\":\"<SS>\",\"id\":301,\"special\":true,\"normalized\":false},"
+ "{\"content\":\"[M]\",\"id\":302,\"special\":true,\"lstrip\":true,\"rstrip\":true},"
+ "{\"content\":\"word\",\"id\":303,\"single_word\":true},"
+ "{\"content\":\"\xC3\xA9" "clair\",\"id\":304,\"normalized\":true},"
+ "{\"content\":\"\xC2\xA7\",\"id\":305},{\"content\":\"!\",\"id\":306}],\"decoder\":{\"type\":\"ByteLevel\"}"));
+ using V = std::vector<int32_t>;
+ EXPECT_EQ(tok.Encode("<SS><S>"), (V{301, 300})) << "the longest token wins at a position";
+ EXPECT_EQ(tok.Encode("a [M] b"), (V{97, 302, 98})) << "lstrip and rstrip absorb the white space";
+ EXPECT_EQ(tok.Encode("word"), (V{303}));
+ auto inside = tok.Encode("swords word.");
+ ASSERT_GE(inside.size(), 3u);
+ EXPECT_EQ(inside[inside.size() - 2], 303) << "a single_word token inside a word is not matched but a free one is";
+ EXPECT_EQ(tok.Encode("\xC3\xA9" "clair"), (V{304}));
+ EXPECT_EQ(tok.Encode("a\xC2\xA7" "b!"), (V{97, 305, 98, 306}));
+ EXPECT_EQ(tok.Encode("<S"), (V{60, 83})) << "a prefix of a token is not the token";
+ EXPECT_EQ(tok.Decode({300, 97, 301}, true), "a");
+ EXPECT_EQ(tok.Decode({300, 97}, false), "<S>a");
+ EXPECT_EQ(tok.SpecialId("<S>"), std::optional<int32_t>(300));
+ EXPECT_FALSE(tok.SpecialId("word").has_value());
+ EXPECT_EQ(tok.SpecialTokens().size(), 3u);
+}
+
+TEST(TokenizerLoader, RolesFromTokenizerConfigAndDirectoryLayout) {
+ std::string base = ::testing::TempDir() + "/laya_tok_roles";
+ auto write = [](const std::string& path, const std::string& content) {
+ std::ofstream(path, std::ios::binary) << content;
+ };
+ std::string cmd = "mkdir -p '" + base + "/m/tokenizer' '" + base + "/plain' '" + base + "/empty'";
+ ASSERT_EQ(std::system(cmd.c_str()), 0);
+ std::string spec = ByteLevelSpec(",\"added_tokens\":[{\"content\":\"<c>\",\"id\":300,\"special\":true},{\"content\":\"<m>\",\"id\":301,\"special\":true}]");
+ write(base + "/plain/tokenizer.json", spec);
+ write(base + "/plain/tokenizer_config.json", "{\"cls_token\":\"<c>\",\"mask_token\":{\"content\":\"<m>\"},\"pad_token\":null}");
+ auto t = Tokenizer::LoadDir(base + "/plain");
+ ASSERT_TRUE(t.ok()) << t.error();
+ EXPECT_EQ(t.value().Cls(), std::optional<int32_t>(300));
+ EXPECT_EQ(t.value().Mask(), std::optional<int32_t>(301));
+ EXPECT_EQ(t.value().MaskToken(), "<m>");
+ EXPECT_FALSE(t.value().Pad().has_value()) << "a null role is absent";
+ EXPECT_FALSE(t.value().Sep().has_value()) << "an unnamed role is absent";
+ write(base + "/plain/tokenizer_config.json", "{\"cls_token\":\"<nope>\"}");
+ EXPECT_FALSE(Tokenizer::LoadDir(base + "/plain").ok()) << "a role naming a token the vocabulary lacks";
+ write(base + "/plain/tokenizer_config.json", "{\"cls_token\":5}");
+ EXPECT_FALSE(Tokenizer::LoadDir(base + "/plain").ok()) << "a role that is not a string or an object";
+ write(base + "/m/tokenizer/tokenizer.json", ByteLevelSpec(""));
+ EXPECT_TRUE(Tokenizer::LoadDir(base + "/m").ok()) << "<dir>/tokenizer/ is preferred";
+ EXPECT_FALSE(Tokenizer::LoadDir(base + "/empty").ok()) << "a directory with no tokenizer";
+}
+
+TEST(TokenizerRefusals, NormalizersPreTokenizersAndRegexDialects) {
+ for (const char* spec : {"{\"type\":\"BertNormalizer\"}", "{}", "{\"type\":\"Replace\",\"pattern\":{\"String\":\"a\"}}",
+ "{\"type\":\"Replace\",\"pattern\":{\"String\":\"\"},\"content\":\"x\"}",
+ "{\"type\":\"Replace\",\"pattern\":{},\"content\":\"x\"}",
+ "{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"(\"},\"content\":\"x\"}", "{\"type\":\"Sequence\"}",
+ "{\"type\":\"Sequence\",\"normalizers\":[1]}"}) {
+ EXPECT_FALSE(tk::RunNormalizerSpec(spec, "a").ok()) << spec;
+ }
+ // The reference reads \s, \d, \w, \b, \p as Unicode; an ASCII engine would differ, so they are refused.
+ for (const char* expr : {"\\\\s+", "a\\\\bb", "[\\\\d]", "\\\\w", "\\\\p{L}"}) {
+ auto r = tk::RunNormalizerSpec(std::string("{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"") + expr + "\"},\"content\":\"x\"}", "a");
+ ASSERT_FALSE(r.ok()) << expr;
+ EXPECT_TRUE(tk::IsUnsupported(r.error())) << expr << ": " << r.error();
+ }
+ auto esc = tk::RunNormalizerSpec("{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"\\\\\\\\\\\\\\\\s\"},\"content\":\"x\"}", "a\\\\s");
+ ASSERT_TRUE(esc.ok()) << esc.error();
+ EXPECT_EQ(esc.value(), "ax") << "an escaped backslash before s is a literal, not a class";
+ auto empty = tk::RunNormalizerSpec("{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"a*\"},\"content\":\"x\"}", "b");
+ ASSERT_FALSE(empty.ok());
+ EXPECT_TRUE(tk::IsUnsupported(empty.error())) << "a pattern that can match nothing";
+ for (const char* spec : {"{\"type\":\"Whitespace\"}", "{}", "{\"type\":\"Metaspace\",\"replacement\":\"ab\"}",
+ "{\"type\":\"Metaspace\",\"prepend_scheme\":\"sometimes\"}",
+ "{\"type\":\"Sequence\",\"pretokenizers\":[1]}"}) {
+ EXPECT_FALSE(tk::RunPreTokenizerSpec(spec, "a").ok()) << spec;
+ }
+ EXPECT_TRUE(tk::IsUnsupported(tk::RunPreTokenizerSpec("{\"type\":\"Whitespace\"}", "a").error()));
+}
+
+TEST(TokenizerRegex, SubsetBehaviour) {
+ auto run = [](const std::string& pattern, const std::string& in, const std::string& to = "_") {
+ auto r = tk::RunNormalizerSpec("{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"" + pattern + "\"},\"content\":\"" + to + "\"}", in);
+ EXPECT_TRUE(r.ok()) << pattern << ": " << r.error();
+ return r.ok() ? r.value() : std::string("<error>");
+ };
+ EXPECT_EQ(run("a|b", "cabd"), "c__d");
+ EXPECT_EQ(run("a+?", "aaa"), "___") << "lazy repeat";
+ EXPECT_EQ(run("a+", "baaac"), "b_c") << "greedy repeat";
+ EXPECT_EQ(run("(ab)+c", "ababc abc"), "_ _");
+ EXPECT_EQ(run("[^a-c]", "abxcy"), "ab_c_");
+ EXPECT_EQ(run("^a", "aa"), "_a");
+ EXPECT_EQ(run("a$", "aa"), "a_");
+ EXPECT_EQ(run("a.c", "abc a\\nc"), "_ a\\nc");
+ EXPECT_EQ(run("x{2,3}", "xxxxxxx"), "__x");
+ EXPECT_EQ(run("(?i)HeLLo", "Hello HELLO hello"), "_ _ _");
+ EXPECT_EQ(run("é+", "ééa"), "_a");
+ EXPECT_EQ(run("(a|ab)(c|bcd)", "abcd"), "_") << "leftmost-first: a then bcd";
+ // The failed-state memo keeps nested repeats linear.
+ EXPECT_EQ(run("(a*)*b", std::string(5000, 'a')), std::string(5000, 'a'));
+}
+
+TEST(TokenizerPre, SequenceAndCornerCases) {
+ auto seq = tk::RunPreTokenizerSpec(
+ "{\"type\":\"Sequence\",\"pretokenizers\":[{\"type\":\"Metaspace\",\"prepend_scheme\":\"never\"},{\"type\":\"ByteLevel\",\"add_prefix_space\":false}]}", "ab cd");
+ ASSERT_TRUE(seq.ok());
+ EXPECT_EQ(Join(seq.value()), Join({"ab", "\xC3\xA2\xC4\xB8\xC4\xA3", "cd"})) << "the crate gives ['ab', 'âĸģ', 'cd']";
+ const char* spec = "{\"type\":\"ByteLevel\",\"add_prefix_space\":false,\"use_regex\":true}";
+ struct Case {
+ const char* in;
+ std::vector<std::string> want;
+ } cases[] = {
+ {"a b\t\tc\n\nd\r\ne f", {"a", "\xC4\xA0", "\xC4\xA0" "b", "\xC4\x89", "\xC4\x89", "c", "\xC4\x8A", "\xC4\x8A", "d", "\xC4\x8D", "\xC4\x8A", "e", "\xC4\xA0\xC4\xA0", "\xC4\xA0" "f"}},
+ {"abc123", {"abc", "123"}}, {"123abc", {"123", "abc"}}, {"a 1 b", {"a", "\xC4\xA0" "1", "\xC4\xA0" "b"}},
+ {"x's value", {"x", "'s", "\xC4\xA0" "value"}}, {"", {}}};
+ for (const auto& c : cases) {
+ auto got = tk::RunPreTokenizerSpec(spec, c.in);
+ ASSERT_TRUE(got.ok());
+ EXPECT_EQ(Join(got.value()), Join(c.want)) << c.in;
+ }
+}
diff --git a/laya-go/scripts/gen_tables.py b/laya-go/scripts/gen_tables.py
index 9aa3a986..8a4ff32b 100644
--- a/laya-go/scripts/gen_tables.py
+++ b/laya-go/scripts/gen_tables.py
@@ -37,6 +37,7 @@ OUT_LANG = os.path.join(GO, "lang", "tables_gen.go")
OUT_TOKENIZER = os.path.join(GO, "tokenizer", "tables_gen.go")
OUT_CPP_UNICODE = os.path.join(REPO, "laya-cpp", "src", "unicodedata_tables.inc")
OUT_CPP_LANG = os.path.join(REPO, "laya-cpp", "src", "lang_tables.inc")
+OUT_CPP_TOKENIZER = os.path.join(REPO, "laya-cpp", "src", "tokenizer_tables.inc")
def wrap(values, indent="\t", width=100):
@@ -74,6 +75,14 @@ def table(name, rows, doc):
doc, len(rows), sum(b - a + 1 for a, b in rows), name, wrap(flat))
+def tok_array(cpp, go_name, cpp_name, values, doc, ctype="uint32_t"):
+ """One generated array of the tokenizer tables, as Go (`var goName = [...]rune{`) or C++."""
+ if cpp:
+ return "// %s\ninline constexpr %s %s[] = {\n%s\n};\n\n" % (doc, ctype, cpp_name, wrap(values, indent=" "))
+ gotype = "uint64" if ctype == "uint64_t" else "uint32" if ctype == "uint32_t" and go_name.endswith("Offsets") else "rune"
+ return "// %s\nvar %s = [...]%s{\n%s\n}\n\n" % (doc, go_name, gotype, wrap(values))
+
+
def ascii_masks(name, predicate):
low = sum(1 << cp for cp in range(0, 64) if predicate(cp))
high = sum(1 << (cp - 64) for cp in range(64, 128) if predicate(cp))
@@ -327,13 +336,26 @@ def render_lang_cpp():
# ---------------------------------------------------------------------------------------------
# The tokenizers crate: pre-tokenizer classes, single_word flank, lowercase, normalization.
# ---------------------------------------------------------------------------------------------
-def render_tokenizer():
+def render_tokenizer(cpp=False):
import tokenizers
from tokenizers import AddedToken, Tokenizer, models, normalizers, pre_tokenizers
- out = [header("tokenizers %s" % tokenizers.__version__), "package tokenizer\n\n"]
- out.append("// Reference names the implementation the tables below were probed from.\n"
- "const Reference = \"tokenizers %s\"\n\n" % tokenizers.__version__)
+ if cpp:
+ out = ["// Code generated by laya-go/scripts/gen_tables.py --lang cpp from tokenizers %s; DO NOT EDIT.\n//\n"
+ "// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp\n\n"
+ "inline constexpr const char* kTokenizerReference = \"tokenizers %s\";\n\n"
+ % (tokenizers.__version__, tokenizers.__version__)]
+ else:
+ out = [header("tokenizers %s" % tokenizers.__version__), "package tokenizer\n\n"]
+ out.append("// Reference names the implementation the tables below were probed from.\n"
+ "const Reference = \"tokenizers %s\"\n\n" % tokenizers.__version__)
+
+ def rng(go_name, cpp_name, rows, doc):
+ if not cpp:
+ return table(go_name, rows, doc)
+ flat = [v for row in rows for v in row]
+ return tok_array(True, go_name, cpp_name, flat, "%s\n// %d ranges covering %d code points."
+ % (doc.replace("\n//", "\n//"), len(rows), sum(b - a + 1 for a, b in rows)))
# Letter / number: "a<cp>" stays ONE piece only if ` ?\p{L}+` consumed the code point;
# "1<cp>" only if ` ?\p{N}+` did. Anything else splits.
@@ -347,9 +369,9 @@ def render_tokenizer():
return False
return test
- out.append(table("letter", ranges(one_piece("a")),
+ out.append(rng("letter", "k_letter", ranges(one_piece("a")),
"letter is the pre-tokenizer pattern's \\p{L}, probed from the reference."))
- out.append(table("number", ranges(one_piece("1")),
+ out.append(rng("number", "k_number", ranges(one_piece("1")),
"number is the pre-tokenizer pattern's \\p{N}, probed from the reference."))
# single_word: an added token is refused when the adjacent character is a word character.
@@ -367,7 +389,7 @@ def render_tokenizer():
raise SystemExit("gen_tables: single_word treats U+%04X differently on each side" % cp)
return left
- out.append(table("wordChar", ranges(flanks),
+ out.append(rng("wordChar", "k_word_char", ranges(flanks),
"wordChar is what the reference counts as a word character beside a\n"
"// single_word added token."))
@@ -384,16 +406,25 @@ def render_tokenizer():
simple.append((cp, ord(mapped)))
else:
multi.append((cp, [ord(c) for c in mapped]))
- out.append("var lowerFrom = [...]rune{\n%s\n}\n\n" % wrap([c for c, _ in simple]))
- out.append("var lowerTo = [...]rune{\n%s\n}\n\n" % wrap([t for _, t in simple]))
- out.append("var lowerMulti = map[rune][]rune{\n%s\n}\n\n"
- % "\n".join("\t%#x: {%s}," % (cp, ", ".join("%#x" % c for c in m)) for cp, m in multi))
-
- out.append(render_normalization(normalizers))
+ if cpp:
+ out.append(tok_array(True, "", "k_lower_from", [c for c, _ in simple], "lowercase, one code point to one"))
+ out.append(tok_array(True, "", "k_lower_to", [t for _, t in simple], "the lowercase of k_lower_from"))
+ assert all(len(m) <= 3 for _, m in multi)
+ out.append("struct TokLowerMulti { uint32_t from; uint32_t count; uint32_t to[3]; };\n"
+ "inline constexpr TokLowerMulti k_lower_multi[] = {\n%s\n};\n\n"
+ % "\n".join(" {%#x, %d, {%s}}," % (cp, len(m), ", ".join("%#x" % c for c in m + [0] * (3 - len(m))))
+ for cp, m in multi))
+ else:
+ out.append("var lowerFrom = [...]rune{\n%s\n}\n\n" % wrap([c for c, _ in simple]))
+ out.append("var lowerTo = [...]rune{\n%s\n}\n\n" % wrap([t for _, t in simple]))
+ out.append("var lowerMulti = map[rune][]rune{\n%s\n}\n\n"
+ % "\n".join("\t%#x: {%s}," % (cp, ", ".join("%#x" % c for c in m)) for cp, m in multi))
+
+ out.append(render_normalization(normalizers, cpp))
return "".join(out)
-def render_normalization(normalizers):
+def render_normalization(normalizers, cpp=False):
nfc, nfd = normalizers.NFC(), normalizers.NFD()
nfkd = normalizers.NFKD()
canon, compat = {}, {}
@@ -484,6 +515,10 @@ def render_normalization(normalizers):
for k in ks:
flat.extend(mapping[k])
offsets.append(len(flat))
+ if cpp:
+ return (tok_array(True, "", "k_%s_keys" % name, ks, doc)
+ + tok_array(True, "", "k_%s_offsets" % name, offsets, "offsets into k_%s_pool" % name)
+ + tok_array(True, "", "k_%s_pool" % name, flat, "the decompositions of k_%s_keys" % name))
return ("// %s\nvar %sKeys = [...]rune{\n%s\n}\n\nvar %sOffsets = [...]uint32{\n%s\n}\n\n"
"var %sPool = [...]rune{\n%s\n}\n\n"
% (doc, name, wrap(ks), name, wrap(offsets), name, wrap(flat)))
@@ -493,18 +528,24 @@ def render_normalization(normalizers):
"// point that has one, except Hangul syllables, which are algorithmic."))
text.append(pool(compat, "compat", "compat* hold the full compatibility decomposition (NFKD) of every code\n"
"// point where it differs from the canonical one."))
- text.append("// cccRanges are (lo, hi, class) triples of the canonical combining class, sorted.\n"
- "var cccRanges = [...]rune{\n%s\n}\n\n" % wrap([v for r in rows for v in r]))
- text.append("// composeKeys are (first<<32 | second) of each primary composition pair, sorted;\n"
- "// composeValues is the composite.\nvar composeKeys = [...]uint64{\n%s\n}\n\n"
- % wrap(["%#x" % ((a << 32) | b) for a, b in keys]))
+ ccc_doc = "cccRanges are (lo, hi, class) triples of the canonical combining class, sorted."
+ comp_doc = "composeKeys are (first<<32 | second) of each primary composition pair, sorted;\n// composeValues is the composite."
+ if cpp:
+ text.append(tok_array(True, "", "k_ccc_ranges", [v for r in rows for v in r], ccc_doc))
+ text.append(tok_array(True, "", "k_compose_keys", ["%#xULL" % ((a << 32) | b) for a, b in keys], comp_doc, "uint64_t"))
+ text.append(tok_array(True, "", "k_compose_values", [pairs[k] for k in keys], "the composite of k_compose_keys"))
+ return "".join(text)
+ text.append("// %s\nvar cccRanges = [...]rune{\n%s\n}\n\n" % (ccc_doc, wrap([v for r in rows for v in r])))
+ text.append("// %s\nvar composeKeys = [...]uint64{\n%s\n}\n\n"
+ % (comp_doc, wrap(["%#x" % ((a << 32) | b) for a, b in keys])))
text.append("var composeValues = [...]rune{\n%s\n}\n" % wrap([pairs[k] for k in keys]))
return "".join(text)
TARGETS = ((OUT_UNICODE, render_unicode), (OUT_LANG, render_lang), (OUT_TOKENIZER, render_tokenizer))
-# laya-cpp: the tokenizer tables join in C2.
-CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)), (OUT_CPP_LANG, render_lang_cpp))
+# laya-cpp generated tables.
+CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)), (OUT_CPP_LANG, render_lang_cpp),
+ (OUT_CPP_TOKENIZER, lambda: render_tokenizer(cpp=True)))
def main(argv=None):