Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(cpp): laya::tokenizer, loader, normalizers, pretokenizers, BPE and decoders
Adeshina Lasisicacf753f33e0
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txtindex af2023d2..0fd7830d 100644--- a/laya-cpp/CMakeLists.txt+++ b/laya-cpp/CMakeLists.txt@@ -25,7 +25,8 @@ function(laya_target_flags target)target_compile_options(${target} PRIVATE ${LAYA_FP_FLAGS} ${LAYA_WARN_FLAGS})endfunction()-add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp src/lang.cpp)+add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp src/lang.cpp+ src/tokenizer.cpp)target_include_directories(laya PUBLIC include)laya_target_flags(laya)@@ -44,7 +45,8 @@ if(LAYA_BUILD_TESTS)FetchContent_MakeAvailable(googletest)enable_testing()- add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp tests/hardening_test.cpp)+ add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp tests/hardening_test.cpp+ tests/tokenizer_test.cpp)target_link_libraries(laya_tests PRIVATE laya GTest::gtest_main)laya_target_flags(laya_tests)diff --git a/laya-cpp/include/laya/tokenizer.hpp b/laya-cpp/include/laya/tokenizer.hppnew file mode 100644index 00000000..72325e7c--- /dev/null+++ b/laya-cpp/include/laya/tokenizer.hpp@@ -0,0 +1,81 @@+#pragma once+// A reader and runner for Hugging Face tokenizer.json files. It reproduces the ids the reference+// `tokenizers` crate (0.23.2) gives for the BPE tokenizers laya ships: the normalizer, the+// pre-tokenizer, the added-token splitter, BPE with its byte fallback, and the decoder chain.+//+// Anything the loader does not implement is refused with an error that begins with kUnsupported+// rather than approximated: an approximated tokenizer changes ids in a way no answer-level test+// localises. A loaded Tokenizer is immutable and safe for concurrent use. Strings are UTF-8; text+// that is not valid UTF-8 is read as bytes.decode("utf-8", "replace") would deliver it.+//+// The Unicode behaviour that decides ids (letter and number classes, word characters beside a+// single_word token, lowercase, normalization) is compiled in from the reference, see+// laya-go/scripts/gen_tables.py --lang cpp, so ids do not depend on the C++ library's Unicode.+#include <cstdint>+#include <map>+#include <memory>+#include <optional>+#include <string>+#include <string_view>+#include <vector>++#include "laya/result.hpp"++namespace laya {+namespace tokenizer {++// Every error that refuses a feature this library does not implement starts with this text.+extern const char* const kUnsupported;+bool IsUnsupported(const std::string& error);++struct Impl;++class Tokenizer {+ public:+ Tokenizer() = default; // empty, only so Result<Tokenizer> can hold one; obtain a usable one from Parse or LoadDir+ // tokenizer.json bytes and, optionally, tokenizer_config.json bytes (names the CLS, SEP, MASK+ // and PAD tokens).+ static Result<Tokenizer> Parse(std::string_view tokenizer_json,+ std::optional<std::string_view> config_json = std::nullopt);+ // <dir>/tokenizer/tokenizer.json when that exists, else <dir>/tokenizer.json, with the+ // tokenizer_config.json beside whichever is used (it may be absent).+ static Result<Tokenizer> LoadDir(const std::string& dir);++ // Token ids of text. limit < 0 means no limit, otherwise only the first limit ids are made.+ std::vector<int32_t> Encode(std::string_view text, int64_t limit = -1) const;+ // Ids the vocabulary does not hold are skipped, as the reference skips them; with+ // skip_special_tokens the added tokens marked special are skipped too.+ std::string Decode(const std::vector<int32_t>& ids, bool skip_special_tokens) const;++ size_t VocabSize() const; // the model vocabulary plus added tokens outside it+ int32_t HighestId() const;+ std::optional<int32_t> TokenToId(std::string_view token) const;+ std::optional<std::string> IdToToken(int32_t id) const;+ std::optional<int32_t> SpecialId(std::string_view content) const;+ std::map<std::string, int32_t> SpecialTokens() const;+ std::optional<int32_t> Cls() const;+ std::optional<int32_t> Sep() const;+ std::optional<int32_t> Mask() const;+ std::optional<int32_t> Pad() const;+ std::string MaskToken() const; // "" when none is configured++ // Test seams: the stages of Encode, so a stage can be compared with the reference on its own.+ std::string NormalizeForTest(std::string_view text) const;+ std::vector<std::string> PreTokenizeForTest(std::string_view normalized) const;++ private:+ explicit Tokenizer(std::shared_ptr<const Impl> impl) : impl_(std::move(impl)) {}+ std::shared_ptr<const Impl> impl_;+};++// Stages exposed for the golden tests that sweep them without a tokenizer.json.+enum class Form { NFC, NFD, NFKC, NFKD };+std::string Normalize(std::string_view text, Form form);+std::string Lowercase(std::string_view text); // the crate's, no final-sigma rule++// A normalizer or pre-tokenizer built from its tokenizer.json section, e.g. {"type":"Lowercase"}.+Result<std::string> RunNormalizerSpec(std::string_view spec_json, std::string_view input);+Result<std::vector<std::string>> RunPreTokenizerSpec(std::string_view spec_json, std::string_view input);++} // namespace tokenizer+} // namespace layadiff --git a/laya-cpp/src/tokenizer.cpp b/laya-cpp/src/tokenizer.cppnew file mode 100644index 00000000..47443fa1--- /dev/null+++ b/laya-cpp/src/tokenizer.cpp@@ -0,0 +1,1758 @@+// Port of laya-go/tokenizer. Internal failures are a private exception caught at the API boundary+// (Parse and the *Spec helpers); no exception crosses the public API.+#include "laya/tokenizer.hpp"++#include <algorithm>+#include <array>+#include <cstdio>+#include <fstream>+#include <functional>+#include <memory>+#include <queue>+#include <sstream>+#include <unordered_map>+#include <unordered_set>++#include "laya/pyvalue.hpp"+#include "laya/unicodedata.hpp"++namespace laya {+namespace tokenizer {++const char* const kUnsupported = "unsupported tokenizer component: ";+bool IsUnsupported(const std::string& e) { return e.rfind(kUnsupported, 0) == 0; }++namespace {+#include "tokenizer_tables.inc"++using pyvalue::Value;+namespace ud = unicodedata;++struct Fail {+ std::string msg;+};+[[noreturn]] void Refuse(const std::string& m) { throw Fail{m}; }+[[noreturn]] void Unsupported(const std::string& m) { throw Fail{std::string(kUnsupported) + m}; }++template <class T, size_t N>+constexpr size_t Len(const T (&)[N]) {+ return N;+}++// ------------------------------------------------------------------------------------------+// Character classes: the reference's, from the generated tables.+// ------------------------------------------------------------------------------------------+bool IsWhiteSpace(char32_t r) {+ switch (r) {+ case '\t': case '\n': case 0x0B: case '\f': case '\r': case ' ': case 0x85: case 0xA0:+ case 0x1680: case 0x2028: case 0x2029: case 0x202F: case 0x205F: case 0x3000:+ return true;+ }+ return r >= 0x2000 && r <= 0x200A;+}++template <size_t N>+bool InRanges(const uint32_t (&pairs)[N], char32_t r) {+ long lo = 0, hi = long(N / 2) - 1;+ while (lo <= hi) {+ long mid = (lo + hi) / 2;+ if (r < pairs[mid * 2]) hi = mid - 1;+ else if (r > pairs[mid * 2 + 1]) lo = mid + 1;+ else return true;+ }+ return false;+}+bool IsLetter(char32_t r) {+ if (r < 0x80) return 'a' <= (r | 0x20) && (r | 0x20) <= 'z';+ return InRanges(k_letter, r);+}+bool IsNumber(char32_t r) {+ if (r < 0x80) return '0' <= r && r <= '9';+ return InRanges(k_number, r);+}+bool IsWordChar(char32_t r) { return InRanges(k_word_char, r); }++// ------------------------------------------------------------------------------------------+// Lowercase (per character, no final sigma) and normalization (UAX #15) from the crate's tables.+// ------------------------------------------------------------------------------------------+void LowerOf(char32_t r, std::string& out) {+ if (r < 0x80) {+ out.push_back(char((r >= 'A' && r <= 'Z') ? r + 32 : r));+ return;+ }+ for (const auto& m : k_lower_multi) {+ if (m.from == r) {+ for (uint32_t i = 0; i < m.count; i++) ud::AppendRune(out, m.to[i]);+ return;+ }+ }+ const uint32_t* end = k_lower_from + Len(k_lower_from);+ const uint32_t* at = std::lower_bound(k_lower_from, end, uint32_t(r));+ if (at != end && *at == r) {+ ud::AppendRune(out, k_lower_to[at - k_lower_from]);+ return;+ }+ ud::AppendRune(out, r);+}++std::vector<char32_t> Runes(std::string_view s) {+ std::vector<char32_t> out;+ out.reserve(s.size());+ while (!s.empty()) {+ auto d = ud::DecodeRune(s);+ out.push_back(d.rune);+ s.remove_prefix(d.width);+ }+ return out;+}+std::string FromRunes(const std::vector<char32_t>& rs) {+ std::string out;+ out.reserve(rs.size());+ for (char32_t r : rs) ud::AppendRune(out, r);+ return out;+}++constexpr char32_t kSBase = 0xAC00, kLBase = 0x1100, kVBase = 0x1161, kTBase = 0x11A7;+constexpr char32_t kLCount = 19, kVCount = 21, kTCount = 28, kNCount = kVCount * kTCount,+ kSCount = kLCount * kNCount;++bool LookupPool(const uint32_t* keys, size_t nkeys, const uint32_t* offsets, const uint32_t* pool,+ char32_t r, const uint32_t** begin, const uint32_t** end) {+ const uint32_t* at = std::lower_bound(keys, keys + nkeys, uint32_t(r));+ if (at == keys + nkeys || *at != r) return false;+ size_t i = size_t(at - keys);+ *begin = pool + offsets[i];+ *end = pool + offsets[i + 1];+ return true;+}++void Decompose(std::vector<char32_t>& dst, char32_t r, bool compat) {+ if (r < 0xA0 || (r < 0xC0 && !compat)) {+ dst.push_back(r);+ return;+ }+ if (r >= kSBase && r < kSBase + kSCount) {+ char32_t s = r - kSBase;+ dst.push_back(kLBase + s / kNCount);+ dst.push_back(kVBase + (s % kNCount) / kTCount);+ if (s % kTCount != 0) dst.push_back(kTBase + s % kTCount);+ return;+ }+ const uint32_t *b, *e;+ if (compat && LookupPool(k_compat_keys, Len(k_compat_keys), k_compat_offsets, k_compat_pool, r, &b, &e)) {+ dst.insert(dst.end(), b, e);+ return;+ }+ if (LookupPool(k_canon_keys, Len(k_canon_keys), k_canon_offsets, k_canon_pool, r, &b, &e)) {+ dst.insert(dst.end(), b, e);+ return;+ }+ dst.push_back(r);+}++uint8_t CombiningClass(char32_t r) {+ if (r < 0x300) return 0;+ size_t n = Len(k_ccc_ranges) / 3, lo = 0, hi = n;+ while (lo < hi) { // first triple whose hi >= r+ size_t mid = (lo + hi) / 2;+ if (k_ccc_ranges[mid * 3 + 1] >= r) hi = mid;+ else lo = mid + 1;+ }+ if (lo < n && k_ccc_ranges[lo * 3] <= r) return uint8_t(k_ccc_ranges[lo * 3 + 2]);+ return 0;+}++void Reorder(std::vector<char32_t>& rs) {+ for (size_t i = 0; i < rs.size();) {+ if (CombiningClass(rs[i]) == 0) {+ i++;+ continue;+ }+ size_t j = i;+ while (j < rs.size() && CombiningClass(rs[j]) != 0) j++;+ std::stable_sort(rs.begin() + long(i), rs.begin() + long(j),+ [](char32_t a, char32_t b) { return CombiningClass(a) < CombiningClass(b); });+ i = j;+ }+}++bool ComposePair(char32_t first, char32_t second, char32_t* out) {+ if (first >= kLBase && first < kLBase + kLCount && second >= kVBase && second < kVBase + kVCount) {+ *out = kSBase + ((first - kLBase) * kVCount + (second - kVBase)) * kTCount;+ return true;+ }+ if (first >= kSBase && first < kSBase + kSCount && (first - kSBase) % kTCount == 0 &&+ second > kTBase && second < kTBase + kTCount) {+ *out = first + (second - kTBase);+ return true;+ }+ uint64_t key = (uint64_t(first) << 32) | second;+ const uint64_t* end = k_compose_keys + Len(k_compose_keys);+ const uint64_t* at = std::lower_bound(k_compose_keys, end, key);+ if (at != end && *at == key) {+ *out = k_compose_values[at - k_compose_keys];+ return true;+ }+ return false;+}++void ComposeRunes(std::vector<char32_t>& rs) {+ std::vector<char32_t> out;+ out.reserve(rs.size());+ long starter = -1;+ uint8_t last = 0;+ for (char32_t r : rs) {+ uint8_t cc = CombiningClass(r);+ char32_t c;+ if (starter >= 0 && (last < cc || last == 0) && ComposePair(out[size_t(starter)], r, &c)) {+ out[size_t(starter)] = c;+ continue;+ }+ if (cc == 0) {+ starter = long(out.size());+ last = 0;+ } else {+ last = cc;+ }+ out.push_back(r);+ }+ rs.swap(out);+}++std::string NormalizeImpl(std::string_view s, Form f) {+ bool compat = f == Form::NFKC || f == Form::NFKD, compose = f == Form::NFC || f == Form::NFKC;+ bool ascii = true, below = true;+ for (char32_t r : Runes(s)) {+ if (r >= 0x80) ascii = false;+ if (r >= 0x300) {+ below = false;+ break;+ }+ }+ // ASCII has no decomposition in any form, and nothing below U+0300 combines with what precedes.+ if (ascii || (below && compose && !compat)) return std::string(s);+ std::vector<char32_t> rs;+ for (char32_t r : Runes(s)) Decompose(rs, r, compat);+ Reorder(rs);+ if (compose) ComposeRunes(rs);+ return FromRunes(rs);+}++std::string LowercaseImpl(std::string_view s) {+ std::string out;+ out.reserve(s.size());+ while (!s.empty()) {+ auto d = ud::DecodeRune(s);+ LowerOf(d.rune, out);+ s.remove_prefix(d.width);+ }+ return out;+}++std::string StripImpl(std::string_view s, bool left, bool right) {+ size_t a = 0, b = s.size();+ if (left) {+ while (a < b) {+ auto d = ud::DecodeRune(s.substr(a, b - a));+ if (!IsWhiteSpace(d.rune)) break;+ a += d.width;+ }+ }+ if (right) {+ while (b > a) {+ auto d = ud::DecodeLastRune(s.substr(a, b - a));+ if (!IsWhiteSpace(d.rune)) break;+ b -= d.width;+ }+ }+ return std::string(s.substr(a, b - a));+}++// ------------------------------------------------------------------------------------------+// A small regex for the Replace normalizer. The reference reads \s \d \w \b \p as Unicode, and a+// stand-in regex engine would read them its own way, so those are refused (as laya-go does); what+// is left is literals, classes of explicit ranges, ., ^, $, groups, | and the greedy and lazy+// repeats. Matching is a backtracker that remembers failed (pc, position) pairs, so it is linear.+// ------------------------------------------------------------------------------------------+class Regex {+ public:+ explicit Regex(const std::string& pattern) { Compile(pattern); }++ // Replaces every non-overlapping match with the literal content.+ std::string ReplaceAll(const std::string& text, const std::string& content) const {+ std::string out;+ size_t copied = 0, from = 0;+ while (from <= text.size()) {+ size_t ms, me;+ if (!Find(text, from, &ms, &me)) break;+ out.append(text, copied, ms - copied);+ out += content;+ copied = me;+ from = me; // matches are never empty: Compile refuses a pattern that can match nothing+ }+ out.append(text, copied, std::string::npos);+ return out;+ }+ bool MatchesEmpty() const {+ size_t s, e;+ return Find("", 0, &s, &e);+ }++ private:+ enum Op { kChar, kAny, kSet, kSplit, kJmp, kBol, kEol, kMatch };+ struct Inst {+ Op op;+ char32_t c = 0;+ int x = 0, y = 0;+ };+ struct Set {+ std::vector<std::pair<char32_t, char32_t>> ranges;+ bool negate = false;+ };+ std::vector<Inst> prog_;+ std::vector<Set> sets_;++ // --- parser: pattern -> instructions, emitted directly from a recursive descent -------------+ std::vector<char32_t> p_;+ size_t at_ = 0;+ bool icase_ = false;++ bool More() const { return at_ < p_.size(); }+ char32_t Peek() const { return p_[at_]; }+ int Emit(Op op, char32_t c = 0, int x = 0, int y = 0) {+ prog_.push_back(Inst{op, c, x, y});+ return int(prog_.size()) - 1;+ }++ void Compile(const std::string& pattern) {+ p_ = Runes(pattern);+ if (p_.size() >= 4 && p_[0] == '(' && p_[1] == '?' && p_[2] == 'i' && p_[3] == ')') {+ for (char32_t r : p_) {+ if (r >= 0x80) Unsupported("a case-insensitive Replace regex with a non-ASCII character is not implemented");+ }+ icase_ = true;+ at_ = 4;+ }+ Alternation();+ if (More()) Refuse("a Replace normalizer's regex: unexpected " + std::string(1, char(Peek())));+ Emit(kMatch);+ if (MatchesEmpty()) {+ Unsupported("this Replace regex can match the empty string, whose replacement rules differ between engines");+ }+ }++ void Alternation() {+ int start = int(prog_.size());+ Concatenation();+ std::vector<int> jumps;+ while (More() && Peek() == '|') {+ at_++;+ // split(start of the left branches, next), the left branch jumps over the right.+ prog_.insert(prog_.begin() + start, Inst{kSplit, 0, 0, 0});+ Shift(start + 1, 1);+ int j = Emit(kJmp);+ jumps.push_back(j);+ prog_[size_t(start)].x = start + 1;+ prog_[size_t(start)].y = int(prog_.size());+ Concatenation();+ }+ for (int j : jumps) prog_[size_t(j)].x = int(prog_.size());+ }++ // Inserting an instruction at `from` moves everything after it, so targets at or past `from`+ // that were emitted before the insertion move by `by`.+ void Shift(int from, int by) {+ for (size_t i = size_t(from); i < prog_.size(); i++) {+ Inst& in = prog_[i];+ if (in.op == kSplit || in.op == kJmp) {+ if (in.x >= from) in.x += by;+ if (in.op == kSplit && in.y >= from) in.y += by;+ }+ }+ }++ void Concatenation() {+ while (More() && Peek() != '|' && Peek() != ')') Repeat();+ }++ void Repeat() {+ int start = int(prog_.size());+ Atom();+ int end = int(prog_.size());+ while (More()) {+ char32_t c = Peek();+ int lo, hi; // hi < 0: unbounded+ if (c == '*') { lo = 0; hi = -1; at_++; }+ else if (c == '+') { lo = 1; hi = -1; at_++; }+ else if (c == '?') { lo = 0; hi = 1; at_++; }+ else if (c == '{') {+ size_t save = at_;+ if (!Counts(&lo, &hi)) { at_ = save; break; }+ } else break;+ bool lazy = More() && Peek() == '?';+ if (lazy) at_++;+ std::vector<Inst> body(prog_.begin() + start, prog_.begin() + end);+ for (Inst& in : body) {+ if (in.op == kSplit || in.op == kJmp) {+ in.x -= start;+ if (in.op == kSplit) in.y -= start;+ }+ }+ prog_.resize(size_t(start));+ if (lo > 1000 || hi > 1000) Refuse("a Replace regex repeat count above 1000");+ auto copy = [&]() {+ int base = int(prog_.size());+ for (Inst in : body) {+ if (in.op == kSplit || in.op == kJmp) {+ in.x += base;+ if (in.op == kSplit) in.y += base;+ }+ prog_.push_back(in);+ }+ };+ for (int i = 0; i < lo; i++) copy();+ if (hi < 0) { // L: split(body, out); body; jmp L+ int l = int(prog_.size());+ int s = Emit(kSplit);+ copy();+ Emit(kJmp, 0, l);+ int out = int(prog_.size());+ prog_[size_t(s)].x = lazy ? out : s + 1;+ prog_[size_t(s)].y = lazy ? s + 1 : out;+ } else {+ std::vector<int> splits;+ for (int i = lo; i < hi; i++) {+ splits.push_back(Emit(kSplit));+ copy();+ }+ int out = int(prog_.size());+ for (int s : splits) {+ prog_[size_t(s)].x = lazy ? out : s + 1;+ prog_[size_t(s)].y = lazy ? s + 1 : out;+ }+ }+ end = int(prog_.size());+ }+ }++ bool Counts(int* lo, int* hi) { // at '{'; false when it is not a repeat count (then a literal)+ at_++;+ auto number = [&](int* v) {+ if (!More() || Peek() < '0' || Peek() > '9') return false;+ long n = 0;+ while (More() && Peek() >= '0' && Peek() <= '9') {+ n = n * 10 + long(Peek() - '0');+ if (n > 100000) n = 100000;+ at_++;+ }+ *v = int(n);+ return true;+ };+ if (!number(lo)) return false;+ *hi = *lo;+ if (More() && Peek() == ',') {+ at_++;+ if (More() && Peek() == '}') *hi = -1;+ else if (!number(hi)) return false;+ }+ if (!More() || Peek() != '}') return false;+ at_++;+ if (*hi >= 0 && *hi < *lo) Refuse("a Replace regex repeat count with its bounds reversed");+ return true;+ }++ void Atom() {+ char32_t c = Peek();+ if (c == '(') {+ at_++;+ if (More() && Peek() == '?') {+ if (at_ + 1 < p_.size() && p_[at_ + 1] == ':') at_ += 2;+ else Unsupported("this Replace regex group syntax (?...) is not implemented");+ }+ Alternation();+ if (!More() || Peek() != ')') Refuse("a Replace normalizer's regex: missing )");+ at_++;+ } else if (c == '[') {+ Class();+ } else if (c == '.') {+ at_++;+ Emit(kAny);+ } else if (c == '^') {+ at_++;+ Emit(kBol);+ } else if (c == '$') {+ at_++;+ Emit(kEol);+ } else if (c == '*' || c == '+' || c == '?') {+ Refuse("a Replace normalizer's regex: nothing to repeat");+ } else if (c == '\\') {+ Literal(Escape());+ } else {+ at_++;+ Literal(c);+ }+ }++ char32_t Escape() { // at '\\'+ at_++;+ if (!More()) Refuse("a Replace normalizer's regex ends in a backslash");+ char32_t c = Peek();+ at_++;+ switch (c) {+ case 't': return '\t';+ case 'n': return '\n';+ case 'r': return '\r';+ case 'f': return '\f';+ case 'v': return 0x0B;+ default: break;+ }+ if (c < 0x80 && !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'))) return c;+ Unsupported("the Replace regex escape \\" + std::string(1, char(c < 0x80 ? c : '?')) + " is not implemented");+ }++ void Literal(char32_t c) {+ if (icase_ && ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) {+ Set s;+ s.ranges = {{c | 0x20, c | 0x20}, {c & ~char32_t(0x20), c & ~char32_t(0x20)}};+ sets_.push_back(s);+ Emit(kSet, 0, int(sets_.size()) - 1);+ } else {+ Emit(kChar, c);+ }+ }++ void Class() { // at '['+ at_++;+ Set s;+ if (More() && Peek() == '^') {+ s.negate = true;+ at_++;+ }+ bool first = true;+ while (true) {+ if (!More()) Refuse("a Replace normalizer's regex: missing ]");+ char32_t c = Peek();+ if (c == ']' && !first) { at_++; break; }+ first = false;+ if (c == '[') Unsupported("a nested [ or a POSIX class in a Replace regex is not implemented");+ char32_t lo = c == '\\' ? Escape() : (at_++, c);+ char32_t hi = lo;+ if (More() && Peek() == '-' && at_ + 1 < p_.size() && p_[at_ + 1] != ']') {+ at_++;+ char32_t h = Peek();+ if (h == '[') Unsupported("a nested [ in a Replace regex is not implemented");+ hi = h == '\\' ? Escape() : (at_++, h);+ if (hi < lo) Refuse("a Replace normalizer's regex: a range with its ends reversed");+ }+ s.ranges.push_back({lo, hi});+ }+ if (icase_) {+ auto more = s.ranges;+ for (auto& r : s.ranges) {+ char32_t a = std::max<char32_t>(r.first, 'a'), b = std::min<char32_t>(r.second, 'z');+ if (a <= b) more.push_back({a & ~char32_t(0x20), b & ~char32_t(0x20)});+ a = std::max<char32_t>(r.first, 'A');+ b = std::min<char32_t>(r.second, 'Z');+ if (a <= b) more.push_back({a | 0x20, b | 0x20});+ }+ s.ranges = more;+ }+ sets_.push_back(s);+ Emit(kSet, 0, int(sets_.size()) - 1);+ }++ // --- matcher --------------------------------------------------------------------------------+ bool Find(const std::string& text, size_t from, size_t* ms, size_t* me) const {+ size_t width = text.size() + 1;+ std::vector<bool> seen(prog_.size() * width, false);+ std::vector<std::pair<int, size_t>> stack;+ for (size_t start = from; start <= text.size();) {+ stack.assign(1, {0, start});+ while (!stack.empty()) {+ auto [pc, pos] = stack.back();+ stack.pop_back();+ while (true) {+ size_t key = size_t(pc) * width + pos;+ if (seen[key]) break;+ seen[key] = true;+ const Inst& in = prog_[size_t(pc)];+ if (in.op == kMatch) {+ *ms = start;+ *me = pos;+ return true;+ }+ if (in.op == kJmp) { pc = in.x; continue; }+ if (in.op == kSplit) {+ stack.push_back({in.y, pos});+ pc = in.x;+ continue;+ }+ if (in.op == kBol) { if (pos != 0) break; pc++; continue; }+ if (in.op == kEol) { if (pos != text.size()) break; pc++; continue; }+ if (pos >= text.size()) break;+ auto d = ud::DecodeRune(std::string_view(text).substr(pos));+ bool ok;+ if (in.op == kChar) ok = d.rune == in.c;+ else if (in.op == kAny) ok = d.rune != '\n';+ else {+ const Set& s = sets_[size_t(in.x)];+ bool hit = false;+ for (auto& r : s.ranges) hit = hit || (d.rune >= r.first && d.rune <= r.second);+ ok = hit != s.negate;+ }+ if (!ok) break;+ pos += d.width;+ pc++;+ }+ }+ start += start < text.size() ? ud::DecodeRune(std::string_view(text).substr(start)).width : 1;+ }+ return false;+ }+};++// ------------------------------------------------------------------------------------------+// Component sections of tokenizer.json.+// ------------------------------------------------------------------------------------------+using Normalizer = std::function<std::string(const std::string&)>;++const Value* Member(const Value& node, const char* key) {+ const Value* v = node.Find(key);+ return (v == nullptr || v->is_null()) ? nullptr : v;+}+std::string Str(const Value& n, const char* key, const std::string& def) {+ const Value* v = Member(n, key);+ if (v == nullptr) return def;+ if (v->kind() != Value::Kind::String) Refuse(std::string("\"") + key + "\" must be a string, found " + pyvalue::TypeName(*v));+ return v->as_string();+}+bool Bool(const Value& n, const char* key, bool def) {+ const Value* v = Member(n, key);+ if (v == nullptr) return def;+ if (v->kind() != Value::Kind::Bool) Refuse(std::string("\"") + key + "\" must be a boolean, found " + pyvalue::TypeName(*v));+ return v->as_bool();+}+int Int(const Value& n, const char* key, int def) {+ const Value* v = Member(n, key);+ if (v == nullptr) return def;+ if (v->kind() == Value::Kind::Int && v->as_int() >= -1000000000 && v->as_int() <= 1000000000) return int(v->as_int());+ if (v->kind() == Value::Kind::Float && v->as_float() == double(int(v->as_float()))) return int(v->as_float());+ Refuse(std::string("\"") + key + "\" must be an integer");+}+const Value& Object(const Value& n, const char* key) {+ const Value* v = n.Find(key);+ if (v == nullptr || v->kind() != Value::Kind::Object) Refuse(std::string("\"") + key + "\" must be an object");+ return *v;+}+const Value& Array(const Value& n, const char* key) {+ const Value* v = n.Find(key);+ if (v == nullptr || v->kind() != Value::Kind::Array) Refuse(std::string("\"") + key + "\" must be an array");+ return *v;+}+std::string TypeOf(const Value& n, const char* what) {+ if (n.kind() != Value::Kind::Object) Refuse(std::string(what) + " must be an object or null");+ std::string kind = Str(n, "type", "");+ if (kind.empty()) Refuse(std::string("a ") + what + " must declare a type");+ return kind;+}++bool DialectDiffers(const std::string& expr, char* which) {+ for (size_t i = 0; i + 1 < expr.size(); i++) {+ if (expr[i] != '\\') continue;+ i++;+ if (std::string("sSdDwWbBpP").find(expr[i]) != std::string::npos) {+ *which = expr[i];+ return true;+ }+ }+ return false;+}++std::string ReplaceAllLiteral(const std::string& s, const std::string& from, const std::string& to) {+ std::string out;+ size_t at = 0;+ while (true) {+ size_t hit = s.find(from, at);+ if (hit == std::string::npos) break;+ out.append(s, at, hit - at);+ out += to;+ at = hit + from.size();+ }+ out.append(s, at, std::string::npos);+ return out;+}++Normalizer ParseNormalizer(const Value* n) {+ if (n == nullptr || n->is_null()) return [](const std::string& s) { return s; };+ std::string kind = TypeOf(*n, "normalizer");+ if (kind == "NFC") return [](const std::string& s) { return NormalizeImpl(s, Form::NFC); };+ if (kind == "NFD") return [](const std::string& s) { return NormalizeImpl(s, Form::NFD); };+ if (kind == "NFKC") return [](const std::string& s) { return NormalizeImpl(s, Form::NFKC); };+ if (kind == "NFKD") return [](const std::string& s) { return NormalizeImpl(s, Form::NFKD); };+ if (kind == "Lowercase") return [](const std::string& s) { return LowercaseImpl(s); };+ if (kind == "Prepend") {+ std::string prefix = Str(*n, "prepend", "");+ return [prefix](const std::string& s) { return s.empty() ? s : prefix + s; }; // non-empty text only+ }+ if (kind == "Strip") {+ bool l = Bool(*n, "strip_left", true), r = Bool(*n, "strip_right", true);+ return [l, r](const std::string& s) { return StripImpl(s, l, r); };+ }+ if (kind == "Replace") {+ const Value& pattern = Object(*n, "pattern");+ const Value* content = n->Find("content");+ if (content == nullptr || content->kind() != Value::Kind::String) Refuse("a Replace normalizer must declare content");+ std::string to = content->as_string();+ if (const Value* lit = pattern.Find("String"); lit != nullptr && lit->kind() == Value::Kind::String) {+ std::string from = lit->as_string();+ if (from.empty()) Refuse("a Replace normalizer cannot match the empty string");+ return [from, to](const std::string& s) { return ReplaceAllLiteral(s, from, to); };+ }+ const Value* rx = pattern.Find("Regex");+ if (rx == nullptr || rx->kind() != Value::Kind::String) {+ Refuse("a Replace normalizer's pattern must be {\"String\": ...} or {\"Regex\": ...}");+ }+ char which;+ if (DialectDiffers(rx->as_string(), &which)) {+ Unsupported(std::string("the Replace regex uses \\") + which ++ ", whose meaning differs between the reference's regex engine (Unicode-aware) and "+ "an ASCII one, so it would normalize differently; refusing to load");+ }+ auto re = std::make_shared<Regex>(rx->as_string());+ return [re, to](const std::string& s) { return re->ReplaceAll(s, to); };+ }+ if (kind == "Sequence") {+ std::vector<Normalizer> steps;+ for (const Value& child : Array(*n, "normalizers").items()) {+ if (child.kind() != Value::Kind::Object) Refuse("a normalizer in a Sequence must be an object");+ steps.push_back(ParseNormalizer(&child));+ }+ return [steps](const std::string& s) {+ std::string cur = s;+ for (const auto& step : steps) cur = step(cur);+ return cur;+ };+ }+ Unsupported("the normalizer \"" + kind + "\" is not implemented; refusing to load rather than produce ids "+ "that are wrong in a way no answer-level test would localise");+}++// The byte-level stand-in alphabet: printable ASCII and Latin-1 bytes map to themselves, the other+// 68 to U+0100 upward.+struct Alphabet {+ char32_t to_rune[256];+ int16_t to_byte[0x144];+ Alphabet() {+ std::fill(std::begin(to_byte), std::end(to_byte), int16_t(-1));+ int lifted = 0;+ for (int b = 0; b < 256; b++) {+ if ((b >= '!' && b <= '~') || (b >= 0xA1 && b <= 0xAC) || (b >= 0xAE && b <= 0xFF)) to_rune[b] = char32_t(b);+ else to_rune[b] = char32_t(256 + lifted++);+ to_byte[to_rune[b]] = int16_t(b);+ }+ }+ int ByteFor(char32_t r) const { return r >= 0x144 ? -1 : to_byte[r]; }+ std::string Encode(std::string_view piece) const {+ std::string out;+ out.reserve(piece.size() + piece.size() / 4);+ for (unsigned char c : piece) ud::AppendRune(out, to_rune[c]);+ return out;+ }+};+const Alphabet& TheAlphabet() {+ static const Alphabet a;+ return a;+}++class PreTokenizer {+ public:+ virtual ~PreTokenizer() = default;+ virtual void Split(std::string_view text, std::vector<std::string>& out) const = 0;+};++class WholeText : public PreTokenizer {+ public:+ void Split(std::string_view text, std::vector<std::string>& out) const override {+ if (!text.empty()) out.emplace_back(text);+ }+};++class ByteLevel : public PreTokenizer {+ public:+ ByteLevel(bool prefix, bool regex) : add_prefix_space_(prefix), use_regex_(regex) {}+ void Split(std::string_view text, std::vector<std::string>& out) const override {+ if (text.empty()) return;+ std::string source(text);+ if (add_prefix_space_ && source[0] != ' ') source = " " + source;+ if (!use_regex_) {+ out.push_back(TheAlphabet().Encode(source));+ return;+ }+ for (size_t at = 0; at < source.size();) {+ size_t end = Cut(source, at);+ out.push_back(TheAlphabet().Encode(std::string_view(source).substr(at, end - at)));+ at = end;+ }+ }++ private:+ // The GPT-2 pattern 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s++ // as a scanner: the pattern has a lookahead, and no standard regex has it with Unicode classes.+ static bool InClass(char32_t r, int branch) {+ if (branch == 0) return IsLetter(r);+ if (branch == 1) return IsNumber(r);+ return !IsWhiteSpace(r) && !IsLetter(r) && !IsNumber(r);+ }+ static size_t RunEnd(const std::string& s, size_t at, int branch) {+ while (at < s.size()) {+ auto d = ud::DecodeRune(std::string_view(s).substr(at));+ if (!InClass(d.rune, branch)) break;+ at += d.width;+ }+ return at;+ }+ static size_t Cut(const std::string& s, size_t from) {+ static const char* const kContractions[] = {"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"};+ std::string_view rest = std::string_view(s).substr(from);+ for (const char* c : kContractions) {+ std::string_view cv(c);+ if (rest.substr(0, cv.size()) == cv) return from + cv.size();+ }+ auto first = ud::DecodeRune(rest);+ for (int branch = 0; branch < 3; branch++) {+ if (first.rune == ' ') {+ size_t after = from + 1;+ if (after < s.size() && InClass(ud::DecodeRune(std::string_view(s).substr(after)).rune, branch)) {+ return RunEnd(s, after, branch);+ }+ } else if (InClass(first.rune, branch)) {+ return RunEnd(s, from, branch);+ }+ }+ if (IsWhiteSpace(first.rune)) {+ size_t end = from;+ while (end < s.size()) {+ auto d = ud::DecodeRune(std::string_view(s).substr(end));+ if (!IsWhiteSpace(d.rune)) break;+ end += d.width;+ }+ if (end >= s.size()) return end;+ // \s+(?!\S) gives the last white-space character back to the next piece.+ size_t w = ud::DecodeLastRune(std::string_view(s).substr(0, end)).width;+ if (end - w > from) return end - w;+ return end; // a single space: \s+(?!\S) fails and \s+ takes it+ }+ return from + first.width;+ }+ bool add_prefix_space_, use_regex_;+};++class Metaspace : public PreTokenizer {+ public:+ Metaspace(std::string replacement, bool prepend, bool split)+ : replacement_(std::move(replacement)), prepend_(prepend), split_(split) {}+ void Split(std::string_view text, std::vector<std::string>& out) const override {+ if (text.empty()) return;+ std::string source = ReplaceAllLiteral(std::string(text), " ", replacement_);+ if (prepend_ && source.compare(0, replacement_.size(), replacement_) != 0) source = replacement_ + source;+ if (!split_) {+ out.push_back(source);+ return;+ }+ size_t start = 0;+ size_t from = ud::DecodeRune(source).width; // a marker at the very start does not split+ while (true) {+ size_t next = source.find(replacement_, from);+ if (next == std::string::npos) break;+ out.push_back(source.substr(start, next - start));+ start = next;+ from = next + replacement_.size();+ }+ if (start < source.size()) out.push_back(source.substr(start));+ }++ private:+ std::string replacement_;+ bool prepend_, split_;+};++class SequencePre : public PreTokenizer {+ public:+ std::vector<std::unique_ptr<PreTokenizer>> steps;+ void Split(std::string_view text, std::vector<std::string>& out) const override {+ std::vector<std::string> current{std::string(text)};+ for (const auto& step : steps) {+ std::vector<std::string> next;+ for (const auto& piece : current) step->Split(piece, next);+ current.swap(next);+ }+ out.insert(out.end(), current.begin(), current.end());+ }+};++std::unique_ptr<PreTokenizer> ParsePreTokenizer(const Value* n) {+ if (n == nullptr || n->is_null()) return std::make_unique<WholeText>();+ std::string kind = TypeOf(*n, "pre_tokenizer");+ if (kind == "ByteLevel") return std::make_unique<ByteLevel>(Bool(*n, "add_prefix_space", true), Bool(*n, "use_regex", true));+ if (kind == "Metaspace") {+ std::string replacement = Str(*n, "replacement", "\xE2\x96\x81");+ if (ud::RuneCount(replacement) != 1) Refuse("a Metaspace replacement must be exactly one code point");+ std::string scheme = Str(*n, "prepend_scheme", "always");+ bool split = Bool(*n, "split", true);+ if (scheme == "always" || scheme == "first") return std::make_unique<Metaspace>(replacement, true, split);+ if (scheme == "never") return std::make_unique<Metaspace>(replacement, false, split);+ Refuse("unknown Metaspace prepend_scheme \"" + scheme + "\"");+ }+ if (kind == "Sequence") {+ auto seq = std::make_unique<SequencePre>();+ for (const Value& child : Array(*n, "pretokenizers").items()) {+ if (child.kind() != Value::Kind::Object) Refuse("a pre-tokenizer in a Sequence must be an object");+ seq->steps.push_back(ParsePreTokenizer(&child));+ }+ return seq;+ }+ Unsupported("the pre-tokenizer \"" + kind + "\" is not implemented; refusing to load rather than produce ids "+ "that are wrong in a way no answer-level test would localise");+}++// ------------------------------------------------------------------------------------------+// Decoders.+// ------------------------------------------------------------------------------------------+class Decoder {+ public:+ virtual ~Decoder() = default;+ virtual void Transform(std::vector<std::string>& tokens) const = 0;+};++class ByteLevelDecoder : public Decoder {+ public:+ // A token holding a character outside the alphabet is taken as plain text; the whole is then+ // decoded as lossy UTF-8.+ void Transform(std::vector<std::string>& tokens) const override {+ std::string raw;+ for (const auto& token : tokens) {+ std::string mapped;+ bool ok = true;+ std::string_view rest(token);+ while (!rest.empty()) {+ auto d = ud::DecodeRune(rest);+ int b = TheAlphabet().ByteFor(d.rune);+ if (b < 0) {+ ok = false;+ break;+ }+ mapped.push_back(char(b));+ rest.remove_prefix(d.width);+ }+ raw += ok ? mapped : token;+ }+ tokens.assign(1, ud::Lossy(raw));+ }+};++class ReplaceDecoder : public Decoder {+ public:+ ReplaceDecoder(std::string p, std::string c) : pattern_(std::move(p)), content_(std::move(c)) {}+ void Transform(std::vector<std::string>& tokens) const override {+ for (auto& t : tokens) t = ReplaceAllLiteral(t, pattern_, content_);+ }++ private:+ std::string pattern_, content_;+};++int HexValue(char c) {+ if (c >= '0' && c <= '9') return c - '0';+ if (c >= 'A' && c <= 'F') return c - 'A' + 10;+ if (c >= 'a' && c <= 'f') return c - 'a' + 10;+ return -1;+}+// "<0xNN>", read as Rust's from_str_radix does: lower-case digits, and a '+' on the high digit+// leaves only the low digit.+bool ByteToken(const std::string& t, unsigned char* out) {+ if (t.size() != 6 || t[0] != '<' || t[1] != '0' || t[2] != 'x' || t[5] != '>') return false;+ if (t[3] == '+') {+ int l = HexValue(t[4]);+ *out = (unsigned char)l;+ return l >= 0;+ }+ int h = HexValue(t[3]), l = HexValue(t[4]);+ if (h < 0 || l < 0) return false;+ *out = (unsigned char)(h << 4 | l);+ return true;+}++class ByteFallbackDecoder : public Decoder {+ public:+ void Transform(std::vector<std::string>& tokens) const override {+ std::vector<std::string> out;+ std::string run;+ auto flush = [&]() {+ if (run.empty()) return;+ if (ud::ValidUtf8(run)) out.push_back(run);+ else for (size_t i = 0; i < run.size(); i++) out.push_back("\xEF\xBF\xBD");+ run.clear();+ };+ for (const auto& token : tokens) {+ unsigned char b;+ if (ByteToken(token, &b)) {+ run.push_back(char(b));+ continue;+ }+ flush();+ out.push_back(token);+ }+ flush();+ tokens.swap(out);+ }+};++class FuseDecoder : public Decoder {+ public:+ void Transform(std::vector<std::string>& tokens) const override {+ std::string all;+ for (const auto& t : tokens) all += t;+ tokens.assign(1, all);+ }+};++class StripDecoder : public Decoder {+ public:+ StripDecoder(std::string c, int start, int stop) : c_(std::move(c)), start_(start), stop_(stop) {}+ void Transform(std::vector<std::string>& tokens) const override {+ for (auto& t : tokens) {+ for (int n = 0; n < start_ && t.compare(0, c_.size(), c_) == 0; n++) t.erase(0, c_.size());+ for (int n = 0; n < stop_ && t.size() >= c_.size() && t.compare(t.size() - c_.size(), c_.size(), c_) == 0; n++) {+ t.erase(t.size() - c_.size());+ }+ }+ }++ private:+ std::string c_;+ int start_, stop_;+};++class SequenceDecoder : public Decoder {+ public:+ std::vector<std::unique_ptr<Decoder>> steps;+ void Transform(std::vector<std::string>& tokens) const override {+ for (const auto& s : steps) s->Transform(tokens);+ }+};++std::unique_ptr<Decoder> ParseDecoder(const Value* n) {+ if (n == nullptr || n->is_null()) return nullptr;+ std::string kind = TypeOf(*n, "decoder");+ if (kind == "ByteLevel") return std::make_unique<ByteLevelDecoder>();+ if (kind == "ByteFallback") return std::make_unique<ByteFallbackDecoder>();+ if (kind == "Fuse") return std::make_unique<FuseDecoder>();+ if (kind == "Replace") {+ const Value* p = n->Find("pattern");+ std::string literal;+ if (p != nullptr && p->kind() == Value::Kind::Object) {+ const Value* s = p->Find("String");+ if (s == nullptr || s->kind() != Value::Kind::String) {+ if (p->Find("Regex") != nullptr) {+ Unsupported("only a literal String pattern is implemented in a Replace decoder; this checkpoint declares a Regex one");+ }+ Refuse("a Replace decoder needs a String pattern");+ }+ literal = s->as_string();+ } else if (p != nullptr && p->kind() == Value::Kind::String) {+ literal = p->as_string();+ } else {+ Refuse("a Replace decoder needs a String pattern");+ }+ if (literal.empty()) Refuse("a Replace decoder cannot match the empty string");+ return std::make_unique<ReplaceDecoder>(literal, Str(*n, "content", ""));+ }+ if (kind == "Strip") {+ std::string c = Str(*n, "content", " ");+ if (ud::RuneCount(c) != 1) Refuse("a Strip decoder's content must be one character");+ return std::make_unique<StripDecoder>(c, Int(*n, "start", 0), Int(*n, "stop", 0));+ }+ if (kind == "Sequence") {+ auto seq = std::make_unique<SequenceDecoder>();+ const Value* steps = n->Find("decoders");+ if (steps == nullptr || steps->kind() != Value::Kind::Array) Refuse("a Sequence decoder needs a decoders array");+ for (const Value& step : steps->items()) {+ if (step.kind() != Value::Kind::Object) Refuse("every Sequence decoder step must be an object");+ seq->steps.push_back(ParseDecoder(&step));+ }+ return seq;+ }+ Unsupported("the decoder \"" + kind + "\" is not implemented (ByteLevel, Replace, ByteFallback, Fuse, Strip and "+ "Sequence are); refusing to load rather than decode it wrongly");+}++// ------------------------------------------------------------------------------------------+// Added tokens.+// ------------------------------------------------------------------------------------------+struct AddedToken {+ std::string content;+ int32_t id;+ bool special, lstrip, rstrip, single_word, normalized;+};++struct Span {+ std::string text;+ int32_t id = 0;+ bool is_token = false;+};++// The added tokens that start with one character, longest first (the reference's leftmost-longest+// match). `second` records which bytes follow that character in any of them, so the common miss is+// rejected without trying each token.+struct Bucket {+ std::vector<const AddedToken*> tokens;+ bool second[256] = {};+ bool whole = false; // a token is that single character+ void Add(const AddedToken* t, size_t first_width) {+ tokens.push_back(t);+ if (t->content.size() == first_width) whole = true;+ else second[(unsigned char)t->content[first_width]] = true;+ }+};++class AddedVocab {+ public:+ bool empty = true;+ AddedVocab() = default;+ explicit AddedVocab(const std::vector<AddedToken>& tokens) : empty(tokens.empty()), store_(tokens) {+ for (const auto& t : store_) {+ if (t.content.empty()) Refuse("an added token cannot have empty content");+ auto d = ud::DecodeRune(t.content);+ if (d.rune < 0x80) ascii_[d.rune].Add(&t, d.width);+ else other_[d.rune].Add(&t, d.width);+ }+ auto longest_first = [](Bucket& b) {+ std::sort(b.tokens.begin(), b.tokens.end(), [](const AddedToken* x, const AddedToken* y) {+ if (x->content.size() != y->content.size()) return x->content.size() > y->content.size();+ return x->content < y->content;+ });+ };+ for (auto& b : ascii_) longest_first(b);+ for (auto& kv : other_) longest_first(kv.second);+ }++ // Cuts text around its added tokens; lstrip and rstrip tokens absorb the white space beside them.+ std::vector<Span> Split(const std::string& text) const {+ std::vector<Span> out;+ if (text.empty()) return out;+ if (empty) {+ out.push_back(Span{text, 0, false});+ return out;+ }+ size_t plain_from = 0;+ for (size_t at = 0; at < text.size();) {+ auto d = ud::DecodeRune(std::string_view(text).substr(at));+ const AddedToken* hit = LongestAt(text, at, d.rune, d.width);+ if (hit == nullptr) {+ at += d.width;+ continue;+ }+ size_t start = at, stop = at + hit->content.size();+ if (hit->lstrip) {+ while (start > plain_from) {+ auto p = ud::DecodeLastRune(std::string_view(text).substr(0, start));+ if (!IsWhiteSpace(p.rune)) break;+ start -= p.width;+ }+ }+ if (hit->rstrip) {+ while (stop < text.size()) {+ auto nx = ud::DecodeRune(std::string_view(text).substr(stop));+ if (!IsWhiteSpace(nx.rune)) break;+ stop += nx.width;+ }+ }+ if (start > plain_from) out.push_back(Span{text.substr(plain_from, start - plain_from), 0, false});+ out.push_back(Span{hit->content, hit->id, true});+ plain_from = at = stop;+ }+ if (plain_from < text.size()) out.push_back(Span{text.substr(plain_from), 0, false});+ return out;+ }++ private:+ const Bucket* BucketOf(char32_t r) const {+ if (r < 0x80) return &ascii_[r];+ auto it = other_.find(r);+ return it == other_.end() ? nullptr : &it->second;+ }+ const AddedToken* LongestAt(const std::string& text, size_t at, char32_t first, size_t width) const {+ const Bucket* b = BucketOf(first);+ if (b == nullptr || b->tokens.empty()) return nullptr;+ if (!b->whole) {+ size_t next = at + width;+ if (next >= text.size() || !b->second[(unsigned char)text[next]]) return nullptr;+ }+ for (const AddedToken* t : b->tokens) {+ if (text.compare(at, t->content.size(), t->content) != 0) continue;+ if (t->single_word && !StandsAlone(text, at, at + t->content.size())) continue;+ return t;+ }+ return nullptr;+ }+ static bool StandsAlone(const std::string& text, size_t start, size_t stop) {+ if (start > 0 && IsWordChar(ud::DecodeLastRune(std::string_view(text).substr(0, start)).rune)) return false;+ if (stop < text.size() && IsWordChar(ud::DecodeRune(std::string_view(text).substr(stop)).rune)) return false;+ return true;+ }+ std::vector<AddedToken> store_; // the buckets point into it; a moved vector keeps its buffer+ Bucket ascii_[128];+ std::unordered_map<char32_t, Bucket> other_;+};++// ------------------------------------------------------------------------------------------+// BPE with byte fallback.+// ------------------------------------------------------------------------------------------+constexpr size_t kHeapMinSymbols = 32;+constexpr int32_t kNoMerge = 1 << 30;+constexpr char32_t kSmallRunes = 0x180;++struct MergeRule {+ int32_t rank = kNoMerge;+ int32_t id = -1;+};+struct Symbol {+ int32_t start, end, id; // id -1: a character the vocabulary lacks+};++uint64_t PairKey(int32_t l, int32_t r) { return (uint64_t(uint32_t(l)) << 32) | uint32_t(r); }++class Bpe {+ public:+ Bpe(const std::unordered_map<std::string, int32_t>& vocab, const std::vector<std::pair<std::string, std::string>>& merges,+ const std::string* unk, bool fuse_unk, bool ignore_merges, bool byte_fallback)+ : vocab_(&vocab), fuse_unk_(fuse_unk), ignore_merges_(ignore_merges) {+ for (char32_t r = 0; r < kSmallRunes; r++) {+ std::string s;+ ud::AppendRune(s, r);+ auto it = vocab.find(s);+ small_[r] = it == vocab.end() ? -1 : it->second;+ }+ if (unk != nullptr) {+ auto it = vocab.find(*unk);+ if (it == vocab.end()) {+ Refuse("the tokenizer names " + *unk + " as its unknown token, but neither the vocabulary nor the added-token "+ "table has an entry for it");+ }+ unk_id_ = it->second;+ has_unk_ = true;+ }+ merges_.reserve(merges.size());+ for (size_t rank = 0; rank < merges.size(); rank++) {+ const auto& pair = merges[rank];+ auto l = vocab.find(pair.first), r = vocab.find(pair.second);+ if (l == vocab.end() || r == vocab.end()) {+ Refuse("merge at rank " + std::to_string(rank) + " uses the symbol \"" + (l == vocab.end() ? pair.first : pair.second) ++ "\", which the vocabulary does not contain; merging on ids would be unsound");+ }+ auto merged = vocab.find(pair.first + pair.second);+ if (merged == vocab.end()) {+ Refuse("merge at rank " + std::to_string(rank) + " produces \"" + pair.first + pair.second ++ "\", which the vocabulary does not contain");+ }+ merges_.emplace(PairKey(l->second, r->second), MergeRule{int32_t(rank), merged->second}); // the lowest rank wins+ }+ if (byte_fallback) {+ has_bytes_ = true;+ for (int v = 0; v < 256; v++) {+ char name[8];+ std::snprintf(name, sizeof name, "<0x%02X>", v);+ auto it = vocab.find(name);+ byte_ids_[v] = it == vocab.end() ? -1 : it->second;+ }+ }+ }++ bool has_unk() const { return has_unk_; }+ bool has_byte_fallback() const { return has_bytes_; }++ void EncodePiece(const std::string& piece, std::vector<int32_t>& out) const {+ if (piece.empty()) return;+ if (ignore_merges_) {+ auto it = vocab_->find(piece);+ if (it != vocab_->end()) {+ out.push_back(it->second);+ return;+ }+ }+ std::vector<Symbol> symbols;+ symbols.reserve(piece.size());+ for (size_t i = 0; i < piece.size();) {+ auto d = ud::DecodeRune(std::string_view(piece).substr(i));+ int32_t id;+ if (d.rune < kSmallRunes && d.rune != ud::kReplacement) {+ id = small_[d.rune];+ } else {+ auto it = vocab_->find(piece.substr(i, d.width));+ id = it == vocab_->end() ? -1 : it->second;+ }+ symbols.push_back(Symbol{int32_t(i), int32_t(i + d.width), id});+ i += d.width;+ }+ if (symbols.size() >= kHeapMinSymbols) MergeWithHeap(symbols);+ else MergeByRescan(symbols);+ Emit(piece, symbols, out);+ }++ // Both strategies are public for the test that they agree.+ void MergeByRescan(std::vector<Symbol>& symbols) const {+ std::vector<MergeRule> rules(symbols.size());+ auto pair = [&](size_t i) { rules[i] = Rule(symbols[i], symbols[i + 1]); };+ for (size_t i = 0; i + 1 < symbols.size(); i++) pair(i);+ while (symbols.size() > 1) {+ int32_t best_rank = kNoMerge;+ long best_at = -1;+ for (size_t i = 0; i + 1 < symbols.size(); i++) {+ if (rules[i].rank < best_rank) {+ best_rank = rules[i].rank;+ best_at = long(i);+ }+ }+ if (best_at < 0) return;+ size_t b = size_t(best_at);+ symbols[b] = Symbol{symbols[b].start, symbols[b + 1].end, rules[b].id};+ symbols.erase(symbols.begin() + long(b) + 1);+ rules.erase(rules.begin() + long(b) + 1);+ if (b + 1 < symbols.size()) pair(b);+ if (b > 0) pair(b - 1);+ }+ }++ void MergeWithHeap(std::vector<Symbol>& symbols) const {+ struct Item {+ int32_t rank, slot, version;+ bool operator>(const Item& o) const { return rank != o.rank ? rank > o.rank : slot > o.slot; }+ };+ size_t n = symbols.size();+ std::vector<int32_t> next(n), prev(n), version(n, 0);+ std::vector<char> dead(n, 0);+ for (size_t i = 0; i < n; i++) {+ prev[i] = int32_t(i) - 1;+ next[i] = int32_t(i) + 1;+ }+ next[n - 1] = -1;+ std::priority_queue<Item, std::vector<Item>, std::greater<Item>> queue;+ auto offer = [&](int32_t slot) {+ int32_t right = next[size_t(slot)];+ if (right < 0) return;+ MergeRule r = Rule(symbols[size_t(slot)], symbols[size_t(right)]);+ if (r.rank != kNoMerge) queue.push(Item{r.rank, slot, version[size_t(slot)]});+ };+ for (size_t i = 0; i < n; i++) offer(int32_t(i));+ while (!queue.empty()) {+ Item item = queue.top();+ queue.pop();+ size_t slot = size_t(item.slot);+ if (dead[slot] || version[slot] != item.version) continue;+ int32_t right = next[slot];+ if (right < 0 || dead[size_t(right)]) continue;+ MergeRule r = Rule(symbols[slot], symbols[size_t(right)]);+ if (r.rank == kNoMerge) continue;+ symbols[slot] = Symbol{symbols[slot].start, symbols[size_t(right)].end, r.id};+ dead[size_t(right)] = 1;+ next[slot] = next[size_t(right)];+ if (next[size_t(right)] >= 0) prev[size_t(next[size_t(right)])] = int32_t(slot);+ version[slot]++;+ offer(int32_t(slot));+ if (prev[slot] >= 0) {+ version[size_t(prev[slot])]++;+ offer(prev[slot]);+ }+ }+ std::vector<Symbol> out;+ for (int32_t i = 0; i >= 0 && size_t(i) < n; i = next[size_t(i)]) {+ if (!dead[size_t(i)]) out.push_back(symbols[size_t(i)]);+ }+ symbols.swap(out);+ }++ private:+ MergeRule Rule(const Symbol& l, const Symbol& r) const {+ if (l.id < 0 || r.id < 0) return MergeRule{};+ auto it = merges_.find(PairKey(l.id, r.id));+ return it == merges_.end() ? MergeRule{} : it->second;+ }++ // A symbol outside the vocabulary becomes its UTF-8 bytes when the checkpoint has byte fallback+ // and every byte resolves, else the unknown token (one per run with fuse_unk), else it is dropped.+ void Emit(const std::string& piece, const std::vector<Symbol>& symbols, std::vector<int32_t>& out) const {+ bool unknown_pending = false;+ for (const Symbol& s : symbols) {+ if (s.id >= 0) {+ unknown_pending = false;+ out.push_back(s.id);+ continue;+ }+ if (has_bytes_) {+ bool resolves = true;+ for (int32_t i = s.start; i < s.end; i++) resolves = resolves && byte_ids_[(unsigned char)piece[size_t(i)]] >= 0;+ if (resolves) {+ for (int32_t i = s.start; i < s.end; i++) out.push_back(byte_ids_[(unsigned char)piece[size_t(i)]]);+ unknown_pending = false;+ continue;+ }+ }+ if (!has_unk_) continue;+ if (fuse_unk_ && unknown_pending) continue;+ out.push_back(unk_id_);+ unknown_pending = true;+ }+ }++ int32_t small_[kSmallRunes];+ const std::unordered_map<std::string, int32_t>* vocab_;+ std::unordered_map<uint64_t, MergeRule> merges_;+ int32_t unk_id_ = 0;+ bool has_unk_ = false, fuse_unk_, ignore_merges_, has_bytes_ = false;+ int32_t byte_ids_[256];+};++} // namespace++// ------------------------------------------------------------------------------------------+// Tokenizer.+// ------------------------------------------------------------------------------------------+struct Impl {+ Normalizer normalizer;+ std::unique_ptr<PreTokenizer> pre;+ std::unique_ptr<Decoder> decoder;+ AddedVocab raw_added, normal_added;+ std::unique_ptr<Bpe> bpe;+ std::unordered_map<std::string, int32_t> ids;+ std::vector<std::string> tokens;+ std::map<std::string, int32_t> specials;+ std::unordered_set<int32_t> is_special;+ std::map<std::string, int32_t> roles;+ std::string mask_token;+ bool byte_level_only = false;+};++namespace {++void ReadMerges(const Value& model, std::vector<std::pair<std::string, std::string>>& out) {+ const Value* merges = model.Find("merges");+ if (merges == nullptr || merges->is_null()) return;+ if (merges->kind() != Value::Kind::Array) Refuse("model.merges must be an array");+ out.reserve(merges->size());+ for (const Value& m : merges->items()) {+ if (m.kind() == Value::Kind::String) {+ const std::string& j = m.as_string();+ bool found = false;+ for (size_t i = 1; i + 1 < j.size(); i++) {+ if (j[i] == ' ') {+ out.emplace_back(j.substr(0, i), j.substr(i + 1));+ found = true;+ break;+ }+ }+ if (!found) Refuse("a space-joined merge must have two non-empty halves, got \"" + j + "\"");+ } else if (m.kind() == Value::Kind::Array && m.size() == 2 && m.items()[0].kind() == Value::Kind::String &&+ m.items()[1].kind() == Value::Kind::String) {+ out.emplace_back(m.items()[0].as_string(), m.items()[1].as_string());+ } else {+ Refuse("a merge must be two strings or one space-joined string");+ }+ }+}++std::shared_ptr<const Impl> Build(std::string_view tokenizer_json, const Value* config) {+ auto parsed = pyvalue::Parse(tokenizer_json);+ if (!parsed.ok()) Refuse("tokenizer.json: " + parsed.error());+ const Value& doc = parsed.value();+ if (doc.kind() != Value::Kind::Object) Refuse("tokenizer.json must be a JSON object");+ const Value* model_v = doc.Find("model");+ if (model_v == nullptr || model_v->kind() != Value::Kind::Object) Refuse("model must be an object");+ const Value& model = *model_v;+ std::string type = Str(model, "type", "");+ if (type != "BPE") {+ Unsupported("this library implements the BPE tokenizer model; the checkpoint declares \"" + type ++ "\". Refusing to load rather than mis-tokenize silently");+ }+ auto impl = std::make_shared<Impl>();+ impl->normalizer = ParseNormalizer(Member(doc, "normalizer"));+ impl->pre = ParsePreTokenizer(Member(doc, "pre_tokenizer"));+ impl->decoder = ParseDecoder(Member(doc, "decoder"));+ impl->byte_level_only = dynamic_cast<const ByteLevel*>(impl->pre.get()) != nullptr;++ // Added tokens.+ std::vector<AddedToken> added;+ if (const Value* at = Member(doc, "added_tokens")) {+ if (at->kind() != Value::Kind::Array) Refuse("added_tokens must be an array");+ std::map<int32_t, std::string> seen;+ for (const Value& r : at->items()) {+ if (r.kind() != Value::Kind::Object) Refuse("an added_tokens entry must be an object");+ std::string content = Str(r, "content", "");+ if (content.empty()) Refuse("an added_tokens entry must declare content");+ const Value* id = Member(r, "id");+ if (id == nullptr || id->kind() != Value::Kind::Int || id->as_int() < 0 || id->as_int() > INT32_MAX) {+ Refuse("added token \"" + content + "\" has no non-negative id");+ }+ int32_t i = int32_t(id->as_int());+ auto clash = seen.find(i);+ if (clash != seen.end() && clash->second != content) {+ Refuse("added token id " + std::to_string(i) + " is claimed by both \"" + clash->second + "\" and \"" + content + "\"");+ }+ seen[i] = content;+ added.push_back(AddedToken{content, i, Bool(r, "special", false), Bool(r, "lstrip", false), Bool(r, "rstrip", false),+ Bool(r, "single_word", false), Bool(r, "normalized", false)});+ }+ }+ const Value* vocab_v = Member(model, "vocab");+ if (vocab_v == nullptr || vocab_v->kind() != Value::Kind::Object) Refuse("model.vocab must be an object");+ auto& vocab = impl->ids;+ vocab.reserve(vocab_v->size() + added.size());+ for (size_t i = 0; i < vocab_v->size(); i++) {+ const Value& v = vocab_v->value(i);+ if (v.kind() != Value::Kind::Int || v.as_int() > INT32_MAX) Refuse("the vocabulary holds an id that is not an integer");+ if (v.as_int() < 0) Refuse("the vocabulary holds a negative id " + std::to_string(v.as_int()));+ vocab[vocab_v->key(i)] = int32_t(v.as_int());+ }+ for (const auto& tok : added) {+ if (tok.special) {+ impl->is_special.insert(tok.id);+ impl->specials[tok.content] = tok.id;+ }+ vocab[tok.content] = tok.id; // the added-token id wins over the vocabulary's, as in the reference+ }+ int32_t highest = 0;+ for (const auto& kv : vocab) highest = std::max(highest, kv.second);+ // Ids may have gaps, but a dense table sized by a hostile id would be gigabytes.+ if (int64_t(highest) >= int64_t(vocab.size()) * 4 + (1 << 16)) {+ Refuse("the highest token id " + std::to_string(highest) + " is far beyond the " + std::to_string(vocab.size()) ++ " tokens; ids this sparse are refused rather than allocated");+ }+ impl->tokens.assign(size_t(highest) + 1, std::string());+ for (const auto& kv : vocab) {+ std::string& owner = impl->tokens[size_t(kv.second)];+ if (!owner.empty() && owner != kv.first) {+ Refuse("the vocabulary gives id " + std::to_string(kv.second) + " to both \"" + owner + "\" and \"" + kv.first + "\"");+ }+ owner = kv.first;+ }++ std::vector<std::pair<std::string, std::string>> merges;+ ReadMerges(model, merges);+ std::string unk;+ const Value* unk_v = Member(model, "unk_token");+ if (unk_v != nullptr && unk_v->kind() != Value::Kind::String) Refuse("model.unk_token must be a string or null");+ if (unk_v != nullptr) unk = unk_v->as_string();+ impl->bpe = std::make_unique<Bpe>(vocab, merges, unk_v != nullptr ? &unk : nullptr, Bool(model, "fuse_unk", false),+ Bool(model, "ignore_merges", false), Bool(model, "byte_fallback", false));++ // Refuse a tokenizer that could drop text without trace.+ if (!impl->bpe->has_unk() && !impl->bpe->has_byte_fallback()) {+ if (!impl->byte_level_only) {+ Unsupported("this checkpoint declares no unk_token and no byte_fallback, and its pre-tokenizer is not byte-level, "+ "so a character outside the vocabulary would be dropped without trace. Refusing to load");+ }+ int missing = 0, reachable = 0;+ std::string first;+ for (int b = 0; b < 256; b++) {+ if (b == 0xC0 || b == 0xC1 || b >= 0xF5) continue; // bytes UTF-8 never contains+ reachable++;+ std::string symbol;+ ud::AppendRune(symbol, TheAlphabet().to_rune[b]);+ if (vocab.find(symbol) == vocab.end()) {+ if (missing == 0) first = symbol;+ missing++;+ }+ }+ if (missing > 0) {+ Refuse("this checkpoint declares no unk_token and no byte_fallback, and " + std::to_string(missing) + " of the " ++ std::to_string(reachable) + " byte-level alphabet symbols that text can reach are absent from its vocabulary "+ "(first: \"" + first + "\"), so some inputs would lose text silently. Refusing to load");+ }+ }++ std::vector<AddedToken> on_raw, on_normalized;+ for (const auto& tok : added) (tok.normalized ? on_normalized : on_raw).push_back(tok);+ impl->raw_added = AddedVocab(on_raw);+ impl->normal_added = AddedVocab(on_normalized);++ // Roles named by tokenizer_config.json.+ if (config != nullptr) {+ static const char* const kRoles[] = {"cls_token", "sep_token", "mask_token", "pad_token", "unk_token"};+ for (const char* key : kRoles) {+ const Value* v = Member(*config, key);+ if (v == nullptr) continue;+ std::string content;+ if (v->kind() == Value::Kind::String) {+ content = v->as_string();+ } else if (v->kind() == Value::Kind::Object && v->Find("content") != nullptr &&+ v->Find("content")->kind() == Value::Kind::String) {+ content = v->Find("content")->as_string();+ } else {+ Refuse(std::string(key) + " in tokenizer_config.json must be a string or an object with a content field");+ }+ auto it = vocab.find(content);+ if (it == vocab.end()) {+ Refuse("tokenizer_config.json names " + content + " as " + key ++ ", but neither the vocabulary nor the added-token table has it");+ }+ impl->roles[key] = it->second;+ if (std::string(key) == "mask_token") impl->mask_token = content;+ }+ }+ return impl;+}++} // namespace++Result<Tokenizer> Tokenizer::Parse(std::string_view tokenizer_json, std::optional<std::string_view> config_json) {+ try {+ Value config;+ const Value* cp = nullptr;+ if (config_json.has_value()) {+ auto c = pyvalue::Parse(*config_json);+ if (!c.ok() || c.value().kind() != Value::Kind::Object) {+ return Result<Tokenizer>::Fail("tokenizer_config.json must be a JSON object" + (c.ok() ? std::string() : ": " + c.error()));+ }+ config = std::move(c).value();+ cp = &config;+ }+ return Result<Tokenizer>::Ok(Tokenizer(Build(tokenizer_json, cp)));+ } catch (const Fail& f) {+ return Result<Tokenizer>::Fail(f.msg);+ } catch (const std::bad_alloc&) {+ return Result<Tokenizer>::Fail("out of memory loading the tokenizer");+ }+}++Result<Tokenizer> Tokenizer::LoadDir(const std::string& dir) {+ auto read = [](const std::string& path, std::string* out) {+ std::ifstream in(path, std::ios::binary);+ if (!in) return false;+ std::stringstream ss;+ ss << in.rdbuf();+ *out = ss.str();+ return true;+ };+ std::string where = dir, json, config;+ if (std::ifstream(dir + "/tokenizer/tokenizer.json").good()) where = dir + "/tokenizer";+ if (!read(where + "/tokenizer.json", &json)) return Result<Tokenizer>::Fail("cannot read " + where + "/tokenizer.json");+ bool has_config = read(where + "/tokenizer_config.json", &config);+ return Parse(json, has_config ? std::optional<std::string_view>(config) : std::nullopt);+}++std::vector<int32_t> Tokenizer::Encode(std::string_view text_in, int64_t limit) const {+ std::vector<int32_t> out;+ if (limit == 0) return out;+ const Impl& t = *impl_;+ std::string text = ud::ToValid(text_in); // a Python str is always valid+ out.reserve(size_t(limit > 0 && limit < int64_t(text.size() / 3 + 16) ? limit : int64_t(text.size() / 3 + 16)));+ auto full = [&]() { return limit >= 0 && int64_t(out.size()) >= limit; };+ auto plain = [&](const std::string& s) {+ std::vector<std::string> pieces;+ t.pre->Split(s, pieces);+ for (const auto& piece : pieces) {+ if (full()) return;+ t.bpe->EncodePiece(piece, out);+ }+ };+ for (const Span& sp : t.raw_added.Split(text)) {+ if (full()) break;+ if (sp.is_token) {+ out.push_back(sp.id);+ continue;+ }+ std::string normalized = t.normalizer(sp.text);+ if (t.normal_added.empty) {+ plain(normalized);+ continue;+ }+ for (const Span& inner : t.normal_added.Split(normalized)) {+ if (full()) break;+ if (inner.is_token) out.push_back(inner.id);+ else plain(inner.text);+ }+ }+ if (limit >= 0 && int64_t(out.size()) > limit) out.resize(size_t(limit));+ return out;+}++std::string Tokenizer::Decode(const std::vector<int32_t>& ids, bool skip_special_tokens) const {+ const Impl& t = *impl_;+ std::vector<std::string> pieces;+ pieces.reserve(ids.size());+ for (int32_t id : ids) {+ if (skip_special_tokens && t.is_special.count(id) != 0) continue;+ if (id < 0 || size_t(id) >= t.tokens.size() || t.tokens[size_t(id)].empty()) continue;+ pieces.push_back(t.tokens[size_t(id)]);+ }+ std::string out;+ if (t.decoder == nullptr) { // with no decoder section the reference joins tokens with a space+ for (size_t i = 0; i < pieces.size(); i++) out += (i ? " " : "") + pieces[i];+ return out;+ }+ t.decoder->Transform(pieces);+ for (const auto& p : pieces) out += p;+ return out;+}++size_t Tokenizer::VocabSize() const { return impl_->ids.size(); }+int32_t Tokenizer::HighestId() const { return int32_t(impl_->tokens.size()) - 1; }+std::optional<int32_t> Tokenizer::TokenToId(std::string_view token) const {+ auto it = impl_->ids.find(std::string(token));+ if (it == impl_->ids.end()) return std::nullopt;+ return it->second;+}+std::optional<std::string> Tokenizer::IdToToken(int32_t id) const {+ if (id < 0 || size_t(id) >= impl_->tokens.size() || impl_->tokens[size_t(id)].empty()) return std::nullopt;+ return impl_->tokens[size_t(id)];+}+std::optional<int32_t> Tokenizer::SpecialId(std::string_view content) const {+ auto it = impl_->specials.find(std::string(content));+ if (it == impl_->specials.end()) return std::nullopt;+ return it->second;+}+std::map<std::string, int32_t> Tokenizer::SpecialTokens() const { return impl_->specials; }+static std::optional<int32_t> Role(const Impl& i, const char* key) {+ auto it = i.roles.find(key);+ if (it == i.roles.end()) return std::nullopt;+ return it->second;+}+std::optional<int32_t> Tokenizer::Cls() const { return Role(*impl_, "cls_token"); }+std::optional<int32_t> Tokenizer::Sep() const { return Role(*impl_, "sep_token"); }+std::optional<int32_t> Tokenizer::Mask() const { return Role(*impl_, "mask_token"); }+std::optional<int32_t> Tokenizer::Pad() const { return Role(*impl_, "pad_token"); }+std::string Tokenizer::MaskToken() const { return impl_->mask_token; }++std::string Tokenizer::NormalizeForTest(std::string_view text) const { return impl_->normalizer(ud::ToValid(text)); }+std::vector<std::string> Tokenizer::PreTokenizeForTest(std::string_view normalized) const {+ std::vector<std::string> out;+ impl_->pre->Split(normalized, out);+ return out;+}++std::string Normalize(std::string_view text, Form form) { return NormalizeImpl(text, form); }+std::string Lowercase(std::string_view text) { return LowercaseImpl(text); }++Result<std::string> RunNormalizerSpec(std::string_view spec_json, std::string_view input) {+ try {+ auto v = pyvalue::Parse(spec_json);+ if (!v.ok()) return Result<std::string>::Fail(v.error());+ return Result<std::string>::Ok(ParseNormalizer(&v.value())(std::string(input)));+ } catch (const Fail& f) {+ return Result<std::string>::Fail(f.msg);+ }+}++Result<std::vector<std::string>> RunPreTokenizerSpec(std::string_view spec_json, std::string_view input) {+ try {+ auto v = pyvalue::Parse(spec_json);+ if (!v.ok()) return Result<std::vector<std::string>>::Fail(v.error());+ std::vector<std::string> out;+ ParsePreTokenizer(&v.value())->Split(input, out);+ return Result<std::vector<std::string>>::Ok(out);+ } catch (const Fail& f) {+ return Result<std::vector<std::string>>::Fail(f.msg);+ }+}++} // namespace tokenizer+} // namespace layadiff --git a/laya-cpp/src/tokenizer_tables.inc b/laya-cpp/src/tokenizer_tables.incnew file mode 100644index 00000000..26233846--- /dev/null+++ b/laya-cpp/src/tokenizer_tables.inc@@ -0,0 +1,2123 @@+// Code generated by laya-go/scripts/gen_tables.py --lang cpp from tokenizers 0.23.2; DO NOT EDIT.+//+// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp++inline constexpr const char* kTokenizerReference = "tokenizers 0.23.2";++// letter is the pre-tokenizer pattern's \p{L}, probed from the reference.+// 677 ranges covering 141028 code points.+inline constexpr uint32_t k_letter[] = {+ 65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736,+ 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, 906, 908, 908,+ 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1488, 1514,+ 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788,+ 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042,+ 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190,+ 2208, 2249, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448,+ 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529,+ 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614,+ 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736,+ 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832,+ 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929,+ 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980,+ 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133,+ 3160, 3162, 3165, 3165, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251,+ 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386,+ 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515,+ 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722,+ 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807,+ 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189,+ 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301,+ 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749,+ 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885,+ 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866,+ 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103,+ 6108, 6108, 6176, 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509,+ 6512, 6516, 6528, 6571, 6576, 6601, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6988,+ 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7306, 7312, 7354,+ 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965,+ 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116,+ 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180,+ 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469,+ 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521,+ 8526, 8526, 8579, 8580, 11264, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559,+ 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696,+ 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823,+ 12293, 12294, 12337, 12341, 12347, 12348, 12353, 12438, 12445, 12447, 12449, 12538, 12540,+ 12543, 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903, 19968, 42124,+ 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653, 42656,+ 42725, 42775, 42783, 42786, 42888, 42891, 42957, 42960, 42961, 42963, 42963, 42965, 42972,+ 42994, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250,+ 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442,+ 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588,+ 43595, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709,+ 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785,+ 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44002,+ 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275,+ 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321,+ 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136,+ 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487,+ 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599,+ 65613, 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66368,+ 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66560, 66717, 66736,+ 66771, 66776, 66811, 66816, 66855, 66864, 66915, 66928, 66938, 66940, 66954, 66956, 66962,+ 66964, 66965, 66967, 66977, 66979, 66993, 66995, 67001, 67003, 67004, 67008, 67059, 67072,+ 67382, 67392, 67413, 67424, 67431, 67456, 67461, 67463, 67504, 67506, 67514, 67584, 67589,+ 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712,+ 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031,+ 68096, 68096, 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288,+ 68295, 68297, 68324, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680,+ 68736, 68786, 68800, 68850, 68864, 68899, 68938, 68965, 68975, 68997, 69248, 69289, 69296,+ 69297, 69314, 69316, 69376, 69404, 69415, 69415, 69424, 69445, 69488, 69505, 69552, 69572,+ 69600, 69622, 69635, 69687, 69745, 69746, 69749, 69749, 69763, 69807, 69840, 69864, 69891,+ 69926, 69956, 69956, 69959, 69959, 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084,+ 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187, 70207, 70208, 70272, 70278, 70280,+ 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416,+ 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, 70493,+ 70497, 70528, 70537, 70539, 70539, 70542, 70542, 70544, 70581, 70583, 70583, 70609, 70609,+ 70611, 70611, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853, 70855,+ 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352,+ 71424, 71450, 71488, 71494, 71680, 71723, 71840, 71903, 71935, 71942, 71945, 71945, 71948,+ 71955, 71957, 71958, 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, 72106, 72144,+ 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284,+ 72329, 72349, 72349, 72368, 72440, 72640, 72672, 72704, 72712, 72714, 72750, 72768, 72768,+ 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, 73063,+ 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488, 73490, 73523,+ 73648, 73648, 73728, 74649, 74880, 75075, 77712, 77808, 77824, 78895, 78913, 78918, 78944,+ 82938, 82944, 83526, 90368, 90397, 92160, 92728, 92736, 92766, 92784, 92862, 92880, 92909,+ 92928, 92975, 92992, 92995, 93027, 93047, 93053, 93071, 93504, 93548, 93760, 93823, 93952,+ 94026, 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101589,+ 101631, 101640, 110576, 110579, 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898,+ 110928, 110930, 110933, 110933, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788,+ 113792, 113800, 113808, 113817, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970,+ 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069,+ 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132,+ 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570,+ 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744,+ 120746, 120770, 120772, 120779, 122624, 122654, 122661, 122666, 122928, 122989, 123136, 123180,+ 123191, 123197, 123214, 123214, 123536, 123565, 123584, 123627, 124112, 124139, 124368, 124397,+ 124400, 124400, 124896, 124902, 124904, 124907, 124909, 124910, 124912, 124926, 124928, 125124,+ 125184, 125251, 125259, 125259, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500,+ 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530,+ 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548,+ 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562,+ 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590,+ 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173791,+ 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 191472, 192093, 194560, 195101,+ 196608, 201546, 201552, 205743,+};++// number is the pre-tokenizer pattern's \p{N}, probed from the reference.+// 144 ranges covering 1911 code points.+inline constexpr uint32_t k_number[] = {+ 48, 57, 178, 179, 185, 185, 188, 190, 1632, 1641, 1776, 1785, 1984, 1993, 2406, 2415, 2534,+ 2543, 2548, 2553, 2662, 2671, 2790, 2799, 2918, 2927, 2930, 2935, 3046, 3058, 3174, 3183, 3192,+ 3198, 3302, 3311, 3416, 3422, 3430, 3448, 3558, 3567, 3664, 3673, 3792, 3801, 3872, 3891, 4160,+ 4169, 4240, 4249, 4969, 4988, 5870, 5872, 6112, 6121, 6128, 6137, 6160, 6169, 6470, 6479, 6608,+ 6618, 6784, 6793, 6800, 6809, 6992, 7001, 7088, 7097, 7232, 7241, 7248, 7257, 8304, 8304, 8308,+ 8313, 8320, 8329, 8528, 8578, 8581, 8585, 9312, 9371, 9450, 9471, 10102, 10131, 11517, 11517,+ 12295, 12295, 12321, 12329, 12344, 12346, 12690, 12693, 12832, 12841, 12872, 12879, 12881,+ 12895, 12928, 12937, 12977, 12991, 42528, 42537, 42726, 42735, 43056, 43061, 43216, 43225,+ 43264, 43273, 43472, 43481, 43504, 43513, 43600, 43609, 44016, 44025, 65296, 65305, 65799,+ 65843, 65856, 65912, 65930, 65931, 66273, 66299, 66336, 66339, 66369, 66369, 66378, 66378,+ 66513, 66517, 66720, 66729, 67672, 67679, 67705, 67711, 67751, 67759, 67835, 67839, 67862,+ 67867, 68028, 68029, 68032, 68047, 68050, 68095, 68160, 68168, 68221, 68222, 68253, 68255,+ 68331, 68335, 68440, 68447, 68472, 68479, 68521, 68527, 68858, 68863, 68912, 68921, 68928,+ 68937, 69216, 69246, 69405, 69414, 69457, 69460, 69573, 69579, 69714, 69743, 69872, 69881,+ 69942, 69951, 70096, 70105, 70113, 70132, 70384, 70393, 70736, 70745, 70864, 70873, 71248,+ 71257, 71360, 71369, 71376, 71395, 71472, 71483, 71904, 71922, 72016, 72025, 72688, 72697,+ 72784, 72812, 73040, 73049, 73120, 73129, 73552, 73561, 73664, 73684, 74752, 74862, 90416,+ 90425, 92768, 92777, 92864, 92873, 93008, 93017, 93019, 93025, 93552, 93561, 93824, 93846,+ 118000, 118009, 119488, 119507, 119520, 119539, 119648, 119672, 120782, 120831, 123200, 123209,+ 123632, 123641, 124144, 124153, 124401, 124410, 125127, 125135, 125264, 125273, 126065, 126123,+ 126125, 126127, 126129, 126132, 126209, 126253, 126255, 126269, 127232, 127244, 130032, 130041,+};++// wordChar is what the reference counts as a word character beside a+// single_word added token.+// 796 ranges covering 144667 code points.+inline constexpr uint32_t k_word_char[] = {+ 48, 57, 65, 90, 95, 95, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705,+ 710, 721, 736, 740, 748, 748, 750, 750, 768, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904,+ 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1155, 1327, 1329, 1366, 1369, 1369, 1376, 1416,+ 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1488, 1514, 1519, 1522, 1552, 1562,+ 1568, 1641, 1646, 1747, 1749, 1756, 1759, 1768, 1770, 1788, 1791, 1791, 1808, 1866, 1869, 1969,+ 1984, 2037, 2042, 2042, 2045, 2045, 2048, 2093, 2112, 2139, 2144, 2154, 2160, 2183, 2185, 2190,+ 2199, 2273, 2275, 2403, 2406, 2415, 2417, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480,+ 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531,+ 2534, 2545, 2556, 2556, 2558, 2558, 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608,+ 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641,+ 2649, 2652, 2654, 2654, 2662, 2677, 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736,+ 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2799,+ 2809, 2815, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873,+ 2876, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2908, 2909, 2911, 2915, 2918, 2927, 2929, 2929,+ 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980,+ 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3055,+ 3072, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3132, 3140, 3142, 3144, 3146, 3149, 3157, 3158,+ 3160, 3162, 3165, 3165, 3168, 3171, 3174, 3183, 3200, 3203, 3205, 3212, 3214, 3216, 3218, 3240,+ 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3293, 3294, 3296, 3299,+ 3302, 3311, 3313, 3315, 3328, 3340, 3342, 3344, 3346, 3396, 3398, 3400, 3402, 3406, 3412, 3415,+ 3423, 3427, 3430, 3439, 3450, 3455, 3457, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517,+ 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3558, 3567, 3570, 3571, 3585, 3642,+ 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3773,+ 3776, 3780, 3782, 3782, 3784, 3790, 3792, 3801, 3804, 3807, 3840, 3840, 3864, 3865, 3872, 3881,+ 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3948, 3953, 3972, 3974, 3991, 3993, 4028,+ 4038, 4038, 4096, 4169, 4176, 4253, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680,+ 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789,+ 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4959,+ 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880,+ 5888, 5909, 5919, 5940, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6099, 6103, 6103,+ 6108, 6109, 6112, 6121, 6155, 6157, 6159, 6169, 6176, 6264, 6272, 6314, 6320, 6389, 6400, 6430,+ 6432, 6443, 6448, 6459, 6470, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6617, 6656, 6683,+ 6688, 6750, 6752, 6780, 6783, 6793, 6800, 6809, 6823, 6823, 6832, 6862, 6912, 6988, 6992, 7001,+ 7019, 7027, 7040, 7155, 7168, 7223, 7232, 7241, 7245, 7293, 7296, 7306, 7312, 7354, 7357, 7359,+ 7376, 7378, 7380, 7418, 7424, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025,+ 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140,+ 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8204, 8205, 8255, 8256, 8276, 8276,+ 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8432, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469,+ 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521,+ 8526, 8526, 8544, 8584, 9398, 9449, 11264, 11492, 11499, 11507, 11520, 11557, 11559, 11559,+ 11565, 11565, 11568, 11623, 11631, 11631, 11647, 11670, 11680, 11686, 11688, 11694, 11696,+ 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11744, 11775,+ 11823, 11823, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12348, 12353, 12438, 12441,+ 12442, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12735,+ 12784, 12799, 13312, 19903, 19968, 42124, 42192, 42237, 42240, 42508, 42512, 42539, 42560,+ 42610, 42612, 42621, 42623, 42737, 42775, 42783, 42786, 42888, 42891, 42957, 42960, 42961,+ 42963, 42963, 42965, 42972, 42994, 43047, 43052, 43052, 43072, 43123, 43136, 43205, 43216,+ 43225, 43232, 43255, 43259, 43259, 43261, 43309, 43312, 43347, 43360, 43388, 43392, 43456,+ 43471, 43481, 43488, 43518, 43520, 43574, 43584, 43597, 43600, 43609, 43616, 43638, 43642,+ 43714, 43739, 43741, 43744, 43759, 43762, 43766, 43777, 43782, 43785, 43790, 43793, 43798,+ 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44010, 44012, 44013, 44016,+ 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262,+ 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323,+ 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65024, 65039,+ 65056, 65071, 65075, 65076, 65101, 65103, 65136, 65140, 65142, 65276, 65296, 65305, 65313,+ 65338, 65343, 65343, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495,+ 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616,+ 65629, 65664, 65786, 65856, 65908, 66045, 66045, 66176, 66204, 66208, 66256, 66272, 66272,+ 66304, 66335, 66349, 66378, 66384, 66426, 66432, 66461, 66464, 66499, 66504, 66511, 66513,+ 66517, 66560, 66717, 66720, 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915,+ 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, 66995,+ 67001, 67003, 67004, 67008, 67059, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461,+ 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644,+ 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861,+ 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68099, 68101, 68102, 68108, 68115, 68117,+ 68119, 68121, 68149, 68152, 68154, 68159, 68159, 68192, 68220, 68224, 68252, 68288, 68295,+ 68297, 68326, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736,+ 68786, 68800, 68850, 68864, 68903, 68912, 68921, 68928, 68965, 68969, 68973, 68975, 68997,+ 69248, 69289, 69291, 69292, 69296, 69297, 69314, 69316, 69372, 69404, 69415, 69415, 69424,+ 69456, 69488, 69509, 69552, 69572, 69600, 69622, 69632, 69702, 69734, 69749, 69759, 69818,+ 69826, 69826, 69840, 69864, 69872, 69881, 69888, 69940, 69942, 69951, 69956, 69959, 69968,+ 70003, 70006, 70006, 70016, 70084, 70089, 70092, 70094, 70106, 70108, 70108, 70144, 70161,+ 70163, 70199, 70206, 70209, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303,+ 70312, 70320, 70378, 70384, 70393, 70400, 70403, 70405, 70412, 70415, 70416, 70419, 70440,+ 70442, 70448, 70450, 70451, 70453, 70457, 70459, 70468, 70471, 70472, 70475, 70477, 70480,+ 70480, 70487, 70487, 70493, 70499, 70502, 70508, 70512, 70516, 70528, 70537, 70539, 70539,+ 70542, 70542, 70544, 70581, 70583, 70592, 70594, 70594, 70597, 70597, 70599, 70602, 70604,+ 70611, 70625, 70626, 70656, 70730, 70736, 70745, 70750, 70753, 70784, 70853, 70855, 70855,+ 70864, 70873, 71040, 71093, 71096, 71104, 71128, 71133, 71168, 71232, 71236, 71236, 71248,+ 71257, 71296, 71352, 71360, 71369, 71376, 71395, 71424, 71450, 71453, 71467, 71472, 71481,+ 71488, 71494, 71680, 71738, 71840, 71913, 71935, 71942, 71945, 71945, 71948, 71955, 71957,+ 71958, 71960, 71989, 71991, 71992, 71995, 72003, 72016, 72025, 72096, 72103, 72106, 72151,+ 72154, 72161, 72163, 72164, 72192, 72254, 72263, 72263, 72272, 72345, 72349, 72349, 72368,+ 72440, 72640, 72672, 72688, 72697, 72704, 72712, 72714, 72758, 72760, 72768, 72784, 72793,+ 72818, 72847, 72850, 72871, 72873, 72886, 72960, 72966, 72968, 72969, 72971, 73014, 73018,+ 73018, 73020, 73021, 73023, 73031, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73102,+ 73104, 73105, 73107, 73112, 73120, 73129, 73440, 73462, 73472, 73488, 73490, 73530, 73534,+ 73538, 73552, 73562, 73648, 73648, 73728, 74649, 74752, 74862, 74880, 75075, 77712, 77808,+ 77824, 78895, 78912, 78933, 78944, 82938, 82944, 83526, 90368, 90425, 92160, 92728, 92736,+ 92766, 92768, 92777, 92784, 92862, 92864, 92873, 92880, 92909, 92912, 92916, 92928, 92982,+ 92992, 92995, 93008, 93017, 93027, 93047, 93053, 93071, 93504, 93548, 93552, 93561, 93760,+ 93823, 93952, 94026, 94031, 94087, 94095, 94111, 94176, 94177, 94179, 94180, 94192, 94193,+ 94208, 100343, 100352, 101589, 101631, 101640, 110576, 110579, 110581, 110587, 110589, 110590,+ 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951, 110960, 111355,+ 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 113821, 113822, 118000, 118009,+ 118528, 118573, 118576, 118598, 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179,+ 119210, 119213, 119362, 119364, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970,+ 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069,+ 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132,+ 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570,+ 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744,+ 120746, 120770, 120772, 120779, 120782, 120831, 121344, 121398, 121403, 121452, 121461, 121461,+ 121476, 121476, 121499, 121503, 121505, 121519, 122624, 122654, 122661, 122666, 122880, 122886,+ 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 122928, 122989, 123023, 123023,+ 123136, 123180, 123184, 123197, 123200, 123209, 123214, 123214, 123536, 123566, 123584, 123641,+ 124112, 124153, 124368, 124410, 124896, 124902, 124904, 124907, 124909, 124910, 124912, 124926,+ 124928, 125124, 125136, 125142, 125184, 125259, 125264, 125273, 126464, 126467, 126469, 126495,+ 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521,+ 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543,+ 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557,+ 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583,+ 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633,+ 126635, 126651, 127280, 127305, 127312, 127337, 127344, 127369, 130032, 130041, 131072, 173791,+ 173824, 177977, 177984, 178205, 178208, 183969, 183984, 191456, 191472, 192093, 194560, 195101,+ 196608, 201546, 201552, 205743, 917760, 917999,+};++// lowercase, one code point to one+inline constexpr uint32_t k_lower_from[] = {+ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,+ 89, 90, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208,+ 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, 221, 222, 256, 258, 260, 262, 264, 266,+ 268, 270, 272, 274, 276, 278, 280, 282, 284, 286, 288, 290, 292, 294, 296, 298, 300, 302, 306,+ 308, 310, 313, 315, 317, 319, 321, 323, 325, 327, 330, 332, 334, 336, 338, 340, 342, 344, 346,+ 348, 350, 352, 354, 356, 358, 360, 362, 364, 366, 368, 370, 372, 374, 376, 377, 379, 381, 385,+ 386, 388, 390, 391, 393, 394, 395, 398, 399, 400, 401, 403, 404, 406, 407, 408, 412, 413, 415,+ 416, 418, 420, 422, 423, 425, 428, 430, 431, 433, 434, 435, 437, 439, 440, 444, 452, 453, 455,+ 456, 458, 459, 461, 463, 465, 467, 469, 471, 473, 475, 478, 480, 482, 484, 486, 488, 490, 492,+ 494, 497, 498, 500, 502, 503, 504, 506, 508, 510, 512, 514, 516, 518, 520, 522, 524, 526, 528,+ 530, 532, 534, 536, 538, 540, 542, 544, 546, 548, 550, 552, 554, 556, 558, 560, 562, 570, 571,+ 573, 574, 577, 579, 580, 581, 582, 584, 586, 588, 590, 880, 882, 886, 895, 902, 904, 905, 906,+ 908, 910, 911, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928,+ 929, 931, 932, 933, 934, 935, 936, 937, 938, 939, 975, 984, 986, 988, 990, 992, 994, 996, 998,+ 1000, 1002, 1004, 1006, 1012, 1015, 1017, 1018, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028,+ 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044,+ 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060,+ 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1120, 1122, 1124, 1126, 1128,+ 1130, 1132, 1134, 1136, 1138, 1140, 1142, 1144, 1146, 1148, 1150, 1152, 1162, 1164, 1166, 1168,+ 1170, 1172, 1174, 1176, 1178, 1180, 1182, 1184, 1186, 1188, 1190, 1192, 1194, 1196, 1198, 1200,+ 1202, 1204, 1206, 1208, 1210, 1212, 1214, 1216, 1217, 1219, 1221, 1223, 1225, 1227, 1229, 1232,+ 1234, 1236, 1238, 1240, 1242, 1244, 1246, 1248, 1250, 1252, 1254, 1256, 1258, 1260, 1262, 1264,+ 1266, 1268, 1270, 1272, 1274, 1276, 1278, 1280, 1282, 1284, 1286, 1288, 1290, 1292, 1294, 1296,+ 1298, 1300, 1302, 1304, 1306, 1308, 1310, 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, 1329,+ 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1345,+ 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361,+ 1362, 1363, 1364, 1365, 1366, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266,+ 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282,+ 4283, 4284, 4285, 4286, 4287, 4288, 4289, 4290, 4291, 4292, 4293, 4295, 4301, 5024, 5025, 5026,+ 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041, 5042,+ 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5054, 5055, 5056, 5057, 5058,+ 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073, 5074,+ 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5089, 5090,+ 5091, 5092, 5093, 5094, 5095, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106,+ 5107, 5108, 5109, 7305, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, 7322, 7323,+ 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339,+ 7340, 7341, 7342, 7343, 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7357,+ 7358, 7359, 7680, 7682, 7684, 7686, 7688, 7690, 7692, 7694, 7696, 7698, 7700, 7702, 7704, 7706,+ 7708, 7710, 7712, 7714, 7716, 7718, 7720, 7722, 7724, 7726, 7728, 7730, 7732, 7734, 7736, 7738,+ 7740, 7742, 7744, 7746, 7748, 7750, 7752, 7754, 7756, 7758, 7760, 7762, 7764, 7766, 7768, 7770,+ 7772, 7774, 7776, 7778, 7780, 7782, 7784, 7786, 7788, 7790, 7792, 7794, 7796, 7798, 7800, 7802,+ 7804, 7806, 7808, 7810, 7812, 7814, 7816, 7818, 7820, 7822, 7824, 7826, 7828, 7838, 7840, 7842,+ 7844, 7846, 7848, 7850, 7852, 7854, 7856, 7858, 7860, 7862, 7864, 7866, 7868, 7870, 7872, 7874,+ 7876, 7878, 7880, 7882, 7884, 7886, 7888, 7890, 7892, 7894, 7896, 7898, 7900, 7902, 7904, 7906,+ 7908, 7910, 7912, 7914, 7916, 7918, 7920, 7922, 7924, 7926, 7928, 7930, 7932, 7934, 7944, 7945,+ 7946, 7947, 7948, 7949, 7950, 7951, 7960, 7961, 7962, 7963, 7964, 7965, 7976, 7977, 7978, 7979,+ 7980, 7981, 7982, 7983, 7992, 7993, 7994, 7995, 7996, 7997, 7998, 7999, 8008, 8009, 8010, 8011,+ 8012, 8013, 8025, 8027, 8029, 8031, 8040, 8041, 8042, 8043, 8044, 8045, 8046, 8047, 8072, 8073,+ 8074, 8075, 8076, 8077, 8078, 8079, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8104, 8105,+ 8106, 8107, 8108, 8109, 8110, 8111, 8120, 8121, 8122, 8123, 8124, 8136, 8137, 8138, 8139, 8140,+ 8152, 8153, 8154, 8155, 8168, 8169, 8170, 8171, 8172, 8184, 8185, 8186, 8187, 8188, 8486, 8490,+ 8491, 8498, 8544, 8545, 8546, 8547, 8548, 8549, 8550, 8551, 8552, 8553, 8554, 8555, 8556, 8557,+ 8558, 8559, 8579, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410,+ 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, 11264, 11265,+ 11266, 11267, 11268, 11269, 11270, 11271, 11272, 11273, 11274, 11275, 11276, 11277, 11278,+ 11279, 11280, 11281, 11282, 11283, 11284, 11285, 11286, 11287, 11288, 11289, 11290, 11291,+ 11292, 11293, 11294, 11295, 11296, 11297, 11298, 11299, 11300, 11301, 11302, 11303, 11304,+ 11305, 11306, 11307, 11308, 11309, 11310, 11311, 11360, 11362, 11363, 11364, 11367, 11369,+ 11371, 11373, 11374, 11375, 11376, 11378, 11381, 11390, 11391, 11392, 11394, 11396, 11398,+ 11400, 11402, 11404, 11406, 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424,+ 11426, 11428, 11430, 11432, 11434, 11436, 11438, 11440, 11442, 11444, 11446, 11448, 11450,+ 11452, 11454, 11456, 11458, 11460, 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476,+ 11478, 11480, 11482, 11484, 11486, 11488, 11490, 11499, 11501, 11506, 42560, 42562, 42564,+ 42566, 42568, 42570, 42572, 42574, 42576, 42578, 42580, 42582, 42584, 42586, 42588, 42590,+ 42592, 42594, 42596, 42598, 42600, 42602, 42604, 42624, 42626, 42628, 42630, 42632, 42634,+ 42636, 42638, 42640, 42642, 42644, 42646, 42648, 42650, 42786, 42788, 42790, 42792, 42794,+ 42796, 42798, 42802, 42804, 42806, 42808, 42810, 42812, 42814, 42816, 42818, 42820, 42822,+ 42824, 42826, 42828, 42830, 42832, 42834, 42836, 42838, 42840, 42842, 42844, 42846, 42848,+ 42850, 42852, 42854, 42856, 42858, 42860, 42862, 42873, 42875, 42877, 42878, 42880, 42882,+ 42884, 42886, 42891, 42893, 42896, 42898, 42902, 42904, 42906, 42908, 42910, 42912, 42914,+ 42916, 42918, 42920, 42922, 42923, 42924, 42925, 42926, 42928, 42929, 42930, 42931, 42932,+ 42934, 42936, 42938, 42940, 42942, 42944, 42946, 42948, 42949, 42950, 42951, 42953, 42955,+ 42956, 42958, 42960, 42962, 42964, 42966, 42968, 42970, 42972, 42997, 65313, 65314, 65315,+ 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323, 65324, 65325, 65326, 65327, 65328,+ 65329, 65330, 65331, 65332, 65333, 65334, 65335, 65336, 65337, 65338, 66560, 66561, 66562,+ 66563, 66564, 66565, 66566, 66567, 66568, 66569, 66570, 66571, 66572, 66573, 66574, 66575,+ 66576, 66577, 66578, 66579, 66580, 66581, 66582, 66583, 66584, 66585, 66586, 66587, 66588,+ 66589, 66590, 66591, 66592, 66593, 66594, 66595, 66596, 66597, 66598, 66599, 66736, 66737,+ 66738, 66739, 66740, 66741, 66742, 66743, 66744, 66745, 66746, 66747, 66748, 66749, 66750,+ 66751, 66752, 66753, 66754, 66755, 66756, 66757, 66758, 66759, 66760, 66761, 66762, 66763,+ 66764, 66765, 66766, 66767, 66768, 66769, 66770, 66771, 66928, 66929, 66930, 66931, 66932,+ 66933, 66934, 66935, 66936, 66937, 66938, 66940, 66941, 66942, 66943, 66944, 66945, 66946,+ 66947, 66948, 66949, 66950, 66951, 66952, 66953, 66954, 66956, 66957, 66958, 66959, 66960,+ 66961, 66962, 66964, 66965, 68736, 68737, 68738, 68739, 68740, 68741, 68742, 68743, 68744,+ 68745, 68746, 68747, 68748, 68749, 68750, 68751, 68752, 68753, 68754, 68755, 68756, 68757,+ 68758, 68759, 68760, 68761, 68762, 68763, 68764, 68765, 68766, 68767, 68768, 68769, 68770,+ 68771, 68772, 68773, 68774, 68775, 68776, 68777, 68778, 68779, 68780, 68781, 68782, 68783,+ 68784, 68785, 68786, 68944, 68945, 68946, 68947, 68948, 68949, 68950, 68951, 68952, 68953,+ 68954, 68955, 68956, 68957, 68958, 68959, 68960, 68961, 68962, 68963, 68964, 68965, 71840,+ 71841, 71842, 71843, 71844, 71845, 71846, 71847, 71848, 71849, 71850, 71851, 71852, 71853,+ 71854, 71855, 71856, 71857, 71858, 71859, 71860, 71861, 71862, 71863, 71864, 71865, 71866,+ 71867, 71868, 71869, 71870, 71871, 93760, 93761, 93762, 93763, 93764, 93765, 93766, 93767,+ 93768, 93769, 93770, 93771, 93772, 93773, 93774, 93775, 93776, 93777, 93778, 93779, 93780,+ 93781, 93782, 93783, 93784, 93785, 93786, 93787, 93788, 93789, 93790, 93791, 93856, 93857,+ 93858, 93859, 93860, 93861, 93862, 93863, 93864, 93865, 93866, 93867, 93868, 93869, 93870,+ 93871, 93872, 93873, 93874, 93875, 93876, 93877, 93878, 93879, 93880, 125184, 125185, 125186,+ 125187, 125188, 125189, 125190, 125191, 125192, 125193, 125194, 125195, 125196, 125197, 125198,+ 125199, 125200, 125201, 125202, 125203, 125204, 125205, 125206, 125207, 125208, 125209, 125210,+ 125211, 125212, 125213, 125214, 125215, 125216, 125217,+};++// the lowercase of k_lower_from+inline constexpr uint32_t k_lower_to[] = {+ 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,+ 116, 117, 118, 119, 120, 121, 122, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235,+ 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 248, 249, 250, 251, 252, 253, 254, 257,+ 259, 261, 263, 265, 267, 269, 271, 273, 275, 277, 279, 281, 283, 285, 287, 289, 291, 293, 295,+ 297, 299, 301, 303, 307, 309, 311, 314, 316, 318, 320, 322, 324, 326, 328, 331, 333, 335, 337,+ 339, 341, 343, 345, 347, 349, 351, 353, 355, 357, 359, 361, 363, 365, 367, 369, 371, 373, 375,+ 255, 378, 380, 382, 595, 387, 389, 596, 392, 598, 599, 396, 477, 601, 603, 402, 608, 611, 617,+ 616, 409, 623, 626, 629, 417, 419, 421, 640, 424, 643, 429, 648, 432, 650, 651, 436, 438, 658,+ 441, 445, 454, 454, 457, 457, 460, 460, 462, 464, 466, 468, 470, 472, 474, 476, 479, 481, 483,+ 485, 487, 489, 491, 493, 495, 499, 499, 501, 405, 447, 505, 507, 509, 511, 513, 515, 517, 519,+ 521, 523, 525, 527, 529, 531, 533, 535, 537, 539, 541, 543, 414, 547, 549, 551, 553, 555, 557,+ 559, 561, 563, 11365, 572, 410, 11366, 578, 384, 649, 652, 583, 585, 587, 589, 591, 881, 883,+ 887, 1011, 940, 941, 942, 943, 972, 973, 974, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954,+ 955, 956, 957, 958, 959, 960, 961, 963, 964, 965, 966, 967, 968, 969, 970, 971, 983, 985, 987,+ 989, 991, 993, 995, 997, 999, 1001, 1003, 1005, 1007, 952, 1016, 1010, 1019, 891, 892, 893,+ 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119,+ 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087,+ 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103,+ 1121, 1123, 1125, 1127, 1129, 1131, 1133, 1135, 1137, 1139, 1141, 1143, 1145, 1147, 1149, 1151,+ 1153, 1163, 1165, 1167, 1169, 1171, 1173, 1175, 1177, 1179, 1181, 1183, 1185, 1187, 1189, 1191,+ 1193, 1195, 1197, 1199, 1201, 1203, 1205, 1207, 1209, 1211, 1213, 1215, 1231, 1218, 1220, 1222,+ 1224, 1226, 1228, 1230, 1233, 1235, 1237, 1239, 1241, 1243, 1245, 1247, 1249, 1251, 1253, 1255,+ 1257, 1259, 1261, 1263, 1265, 1267, 1269, 1271, 1273, 1275, 1277, 1279, 1281, 1283, 1285, 1287,+ 1289, 1291, 1293, 1295, 1297, 1299, 1301, 1303, 1305, 1307, 1309, 1311, 1313, 1315, 1317, 1319,+ 1321, 1323, 1325, 1327, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388,+ 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404,+ 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, 1414, 11520, 11521, 11522, 11523, 11524,+ 11525, 11526, 11527, 11528, 11529, 11530, 11531, 11532, 11533, 11534, 11535, 11536, 11537,+ 11538, 11539, 11540, 11541, 11542, 11543, 11544, 11545, 11546, 11547, 11548, 11549, 11550,+ 11551, 11552, 11553, 11554, 11555, 11556, 11557, 11559, 11565, 43888, 43889, 43890, 43891,+ 43892, 43893, 43894, 43895, 43896, 43897, 43898, 43899, 43900, 43901, 43902, 43903, 43904,+ 43905, 43906, 43907, 43908, 43909, 43910, 43911, 43912, 43913, 43914, 43915, 43916, 43917,+ 43918, 43919, 43920, 43921, 43922, 43923, 43924, 43925, 43926, 43927, 43928, 43929, 43930,+ 43931, 43932, 43933, 43934, 43935, 43936, 43937, 43938, 43939, 43940, 43941, 43942, 43943,+ 43944, 43945, 43946, 43947, 43948, 43949, 43950, 43951, 43952, 43953, 43954, 43955, 43956,+ 43957, 43958, 43959, 43960, 43961, 43962, 43963, 43964, 43965, 43966, 43967, 5112, 5113, 5114,+ 5115, 5116, 5117, 7306, 4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315,+ 4316, 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325, 4326, 4327, 4328, 4329, 4330, 4331,+ 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4349,+ 4350, 4351, 7681, 7683, 7685, 7687, 7689, 7691, 7693, 7695, 7697, 7699, 7701, 7703, 7705, 7707,+ 7709, 7711, 7713, 7715, 7717, 7719, 7721, 7723, 7725, 7727, 7729, 7731, 7733, 7735, 7737, 7739,+ 7741, 7743, 7745, 7747, 7749, 7751, 7753, 7755, 7757, 7759, 7761, 7763, 7765, 7767, 7769, 7771,+ 7773, 7775, 7777, 7779, 7781, 7783, 7785, 7787, 7789, 7791, 7793, 7795, 7797, 7799, 7801, 7803,+ 7805, 7807, 7809, 7811, 7813, 7815, 7817, 7819, 7821, 7823, 7825, 7827, 7829, 223, 7841, 7843,+ 7845, 7847, 7849, 7851, 7853, 7855, 7857, 7859, 7861, 7863, 7865, 7867, 7869, 7871, 7873, 7875,+ 7877, 7879, 7881, 7883, 7885, 7887, 7889, 7891, 7893, 7895, 7897, 7899, 7901, 7903, 7905, 7907,+ 7909, 7911, 7913, 7915, 7917, 7919, 7921, 7923, 7925, 7927, 7929, 7931, 7933, 7935, 7936, 7937,+ 7938, 7939, 7940, 7941, 7942, 7943, 7952, 7953, 7954, 7955, 7956, 7957, 7968, 7969, 7970, 7971,+ 7972, 7973, 7974, 7975, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991, 8000, 8001, 8002, 8003,+ 8004, 8005, 8017, 8019, 8021, 8023, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, 8064, 8065,+ 8066, 8067, 8068, 8069, 8070, 8071, 8080, 8081, 8082, 8083, 8084, 8085, 8086, 8087, 8096, 8097,+ 8098, 8099, 8100, 8101, 8102, 8103, 8112, 8113, 8048, 8049, 8115, 8050, 8051, 8052, 8053, 8131,+ 8144, 8145, 8054, 8055, 8160, 8161, 8058, 8059, 8165, 8056, 8057, 8060, 8061, 8179, 969, 107,+ 229, 8526, 8560, 8561, 8562, 8563, 8564, 8565, 8566, 8567, 8568, 8569, 8570, 8571, 8572, 8573,+ 8574, 8575, 8580, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436,+ 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9448, 9449, 11312, 11313,+ 11314, 11315, 11316, 11317, 11318, 11319, 11320, 11321, 11322, 11323, 11324, 11325, 11326,+ 11327, 11328, 11329, 11330, 11331, 11332, 11333, 11334, 11335, 11336, 11337, 11338, 11339,+ 11340, 11341, 11342, 11343, 11344, 11345, 11346, 11347, 11348, 11349, 11350, 11351, 11352,+ 11353, 11354, 11355, 11356, 11357, 11358, 11359, 11361, 619, 7549, 637, 11368, 11370, 11372,+ 593, 625, 592, 594, 11379, 11382, 575, 576, 11393, 11395, 11397, 11399, 11401, 11403, 11405,+ 11407, 11409, 11411, 11413, 11415, 11417, 11419, 11421, 11423, 11425, 11427, 11429, 11431,+ 11433, 11435, 11437, 11439, 11441, 11443, 11445, 11447, 11449, 11451, 11453, 11455, 11457,+ 11459, 11461, 11463, 11465, 11467, 11469, 11471, 11473, 11475, 11477, 11479, 11481, 11483,+ 11485, 11487, 11489, 11491, 11500, 11502, 11507, 42561, 42563, 42565, 42567, 42569, 42571,+ 42573, 42575, 42577, 42579, 42581, 42583, 42585, 42587, 42589, 42591, 42593, 42595, 42597,+ 42599, 42601, 42603, 42605, 42625, 42627, 42629, 42631, 42633, 42635, 42637, 42639, 42641,+ 42643, 42645, 42647, 42649, 42651, 42787, 42789, 42791, 42793, 42795, 42797, 42799, 42803,+ 42805, 42807, 42809, 42811, 42813, 42815, 42817, 42819, 42821, 42823, 42825, 42827, 42829,+ 42831, 42833, 42835, 42837, 42839, 42841, 42843, 42845, 42847, 42849, 42851, 42853, 42855,+ 42857, 42859, 42861, 42863, 42874, 42876, 7545, 42879, 42881, 42883, 42885, 42887, 42892, 613,+ 42897, 42899, 42903, 42905, 42907, 42909, 42911, 42913, 42915, 42917, 42919, 42921, 614, 604,+ 609, 620, 618, 670, 647, 669, 43859, 42933, 42935, 42937, 42939, 42941, 42943, 42945, 42947,+ 42900, 642, 7566, 42952, 42954, 612, 42957, 42959, 42961, 42963, 42965, 42967, 42969, 42971,+ 411, 42998, 65345, 65346, 65347, 65348, 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356,+ 65357, 65358, 65359, 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, 65369,+ 65370, 66600, 66601, 66602, 66603, 66604, 66605, 66606, 66607, 66608, 66609, 66610, 66611,+ 66612, 66613, 66614, 66615, 66616, 66617, 66618, 66619, 66620, 66621, 66622, 66623, 66624,+ 66625, 66626, 66627, 66628, 66629, 66630, 66631, 66632, 66633, 66634, 66635, 66636, 66637,+ 66638, 66639, 66776, 66777, 66778, 66779, 66780, 66781, 66782, 66783, 66784, 66785, 66786,+ 66787, 66788, 66789, 66790, 66791, 66792, 66793, 66794, 66795, 66796, 66797, 66798, 66799,+ 66800, 66801, 66802, 66803, 66804, 66805, 66806, 66807, 66808, 66809, 66810, 66811, 66967,+ 66968, 66969, 66970, 66971, 66972, 66973, 66974, 66975, 66976, 66977, 66979, 66980, 66981,+ 66982, 66983, 66984, 66985, 66986, 66987, 66988, 66989, 66990, 66991, 66992, 66993, 66995,+ 66996, 66997, 66998, 66999, 67000, 67001, 67003, 67004, 68800, 68801, 68802, 68803, 68804,+ 68805, 68806, 68807, 68808, 68809, 68810, 68811, 68812, 68813, 68814, 68815, 68816, 68817,+ 68818, 68819, 68820, 68821, 68822, 68823, 68824, 68825, 68826, 68827, 68828, 68829, 68830,+ 68831, 68832, 68833, 68834, 68835, 68836, 68837, 68838, 68839, 68840, 68841, 68842, 68843,+ 68844, 68845, 68846, 68847, 68848, 68849, 68850, 68976, 68977, 68978, 68979, 68980, 68981,+ 68982, 68983, 68984, 68985, 68986, 68987, 68988, 68989, 68990, 68991, 68992, 68993, 68994,+ 68995, 68996, 68997, 71872, 71873, 71874, 71875, 71876, 71877, 71878, 71879, 71880, 71881,+ 71882, 71883, 71884, 71885, 71886, 71887, 71888, 71889, 71890, 71891, 71892, 71893, 71894,+ 71895, 71896, 71897, 71898, 71899, 71900, 71901, 71902, 71903, 93792, 93793, 93794, 93795,+ 93796, 93797, 93798, 93799, 93800, 93801, 93802, 93803, 93804, 93805, 93806, 93807, 93808,+ 93809, 93810, 93811, 93812, 93813, 93814, 93815, 93816, 93817, 93818, 93819, 93820, 93821,+ 93822, 93823, 93883, 93884, 93885, 93886, 93887, 93888, 93889, 93890, 93891, 93892, 93893,+ 93894, 93895, 93896, 93897, 93898, 93899, 93900, 93901, 93902, 93903, 93904, 93905, 93906,+ 93907, 125218, 125219, 125220, 125221, 125222, 125223, 125224, 125225, 125226, 125227, 125228,+ 125229, 125230, 125231, 125232, 125233, 125234, 125235, 125236, 125237, 125238, 125239, 125240,+ 125241, 125242, 125243, 125244, 125245, 125246, 125247, 125248, 125249, 125250, 125251,+};++struct TokLowerMulti { uint32_t from; uint32_t count; uint32_t to[3]; };+inline constexpr TokLowerMulti k_lower_multi[] = {+ {0x130, 2, {0x69, 0x307, 0x0}},+};++// canon* hold the full canonical decomposition (NFD) of every code+// point that has one, except Hangul syllables, which are algorithmic.+inline constexpr uint32_t k_canon_keys[] = {+ 192, 193, 194, 195, 196, 197, 199, 200, 201, 202, 203, 204, 205, 206, 207, 209, 210, 211, 212,+ 213, 214, 217, 218, 219, 220, 221, 224, 225, 226, 227, 228, 229, 231, 232, 233, 234, 235, 236,+ 237, 238, 239, 241, 242, 243, 244, 245, 246, 249, 250, 251, 252, 253, 255, 256, 257, 258, 259,+ 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 274, 275, 276, 277, 278, 279, 280,+ 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 296, 297, 298, 299, 300, 301,+ 302, 303, 304, 308, 309, 310, 311, 313, 314, 315, 316, 317, 318, 323, 324, 325, 326, 327, 328,+ 332, 333, 334, 335, 336, 337, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352,+ 353, 354, 355, 356, 357, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373,+ 374, 375, 376, 377, 378, 379, 380, 381, 382, 416, 417, 431, 432, 461, 462, 463, 464, 465, 466,+ 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 478, 479, 480, 481, 482, 483, 486, 487, 488,+ 489, 490, 491, 492, 493, 494, 495, 496, 500, 501, 504, 505, 506, 507, 508, 509, 510, 511, 512,+ 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531,+ 532, 533, 534, 535, 536, 537, 538, 539, 542, 543, 550, 551, 552, 553, 554, 555, 556, 557, 558,+ 559, 560, 561, 562, 563, 832, 833, 835, 836, 884, 894, 901, 902, 903, 904, 905, 906, 908, 910,+ 911, 912, 938, 939, 940, 941, 942, 943, 944, 970, 971, 972, 973, 974, 979, 980, 1024, 1025,+ 1027, 1031, 1036, 1037, 1038, 1049, 1081, 1104, 1105, 1107, 1111, 1116, 1117, 1118, 1142, 1143,+ 1217, 1218, 1232, 1233, 1234, 1235, 1238, 1239, 1242, 1243, 1244, 1245, 1246, 1247, 1250, 1251,+ 1252, 1253, 1254, 1255, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269,+ 1272, 1273, 1570, 1571, 1572, 1573, 1574, 1728, 1730, 1747, 2345, 2353, 2356, 2392, 2393, 2394,+ 2395, 2396, 2397, 2398, 2399, 2507, 2508, 2524, 2525, 2527, 2611, 2614, 2649, 2650, 2651, 2654,+ 2888, 2891, 2892, 2908, 2909, 2964, 3018, 3019, 3020, 3144, 3264, 3271, 3272, 3274, 3275, 3402,+ 3403, 3404, 3546, 3548, 3549, 3550, 3907, 3917, 3922, 3927, 3932, 3945, 3955, 3957, 3958, 3960,+ 3969, 3987, 3997, 4002, 4007, 4012, 4025, 4134, 6918, 6920, 6922, 6924, 6926, 6930, 6971, 6973,+ 6976, 6977, 6979, 7680, 7681, 7682, 7683, 7684, 7685, 7686, 7687, 7688, 7689, 7690, 7691, 7692,+ 7693, 7694, 7695, 7696, 7697, 7698, 7699, 7700, 7701, 7702, 7703, 7704, 7705, 7706, 7707, 7708,+ 7709, 7710, 7711, 7712, 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724,+ 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, 7737, 7738, 7739, 7740,+ 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756,+ 7757, 7758, 7759, 7760, 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772,+ 7773, 7774, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, 7785, 7786, 7787, 7788,+ 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, 7797, 7798, 7799, 7800, 7801, 7802, 7803, 7804,+ 7805, 7806, 7807, 7808, 7809, 7810, 7811, 7812, 7813, 7814, 7815, 7816, 7817, 7818, 7819, 7820,+ 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, 7833, 7835, 7840, 7841,+ 7842, 7843, 7844, 7845, 7846, 7847, 7848, 7849, 7850, 7851, 7852, 7853, 7854, 7855, 7856, 7857,+ 7858, 7859, 7860, 7861, 7862, 7863, 7864, 7865, 7866, 7867, 7868, 7869, 7870, 7871, 7872, 7873,+ 7874, 7875, 7876, 7877, 7878, 7879, 7880, 7881, 7882, 7883, 7884, 7885, 7886, 7887, 7888, 7889,+ 7890, 7891, 7892, 7893, 7894, 7895, 7896, 7897, 7898, 7899, 7900, 7901, 7902, 7903, 7904, 7905,+ 7906, 7907, 7908, 7909, 7910, 7911, 7912, 7913, 7914, 7915, 7916, 7917, 7918, 7919, 7920, 7921,+ 7922, 7923, 7924, 7925, 7926, 7927, 7928, 7929, 7936, 7937, 7938, 7939, 7940, 7941, 7942, 7943,+ 7944, 7945, 7946, 7947, 7948, 7949, 7950, 7951, 7952, 7953, 7954, 7955, 7956, 7957, 7960, 7961,+ 7962, 7963, 7964, 7965, 7968, 7969, 7970, 7971, 7972, 7973, 7974, 7975, 7976, 7977, 7978, 7979,+ 7980, 7981, 7982, 7983, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991, 7992, 7993, 7994, 7995,+ 7996, 7997, 7998, 7999, 8000, 8001, 8002, 8003, 8004, 8005, 8008, 8009, 8010, 8011, 8012, 8013,+ 8016, 8017, 8018, 8019, 8020, 8021, 8022, 8023, 8025, 8027, 8029, 8031, 8032, 8033, 8034, 8035,+ 8036, 8037, 8038, 8039, 8040, 8041, 8042, 8043, 8044, 8045, 8046, 8047, 8048, 8049, 8050, 8051,+ 8052, 8053, 8054, 8055, 8056, 8057, 8058, 8059, 8060, 8061, 8064, 8065, 8066, 8067, 8068, 8069,+ 8070, 8071, 8072, 8073, 8074, 8075, 8076, 8077, 8078, 8079, 8080, 8081, 8082, 8083, 8084, 8085,+ 8086, 8087, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8096, 8097, 8098, 8099, 8100, 8101,+ 8102, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111, 8112, 8113, 8114, 8115, 8116, 8118,+ 8119, 8120, 8121, 8122, 8123, 8124, 8126, 8129, 8130, 8131, 8132, 8134, 8135, 8136, 8137, 8138,+ 8139, 8140, 8141, 8142, 8143, 8144, 8145, 8146, 8147, 8150, 8151, 8152, 8153, 8154, 8155, 8157,+ 8158, 8159, 8160, 8161, 8162, 8163, 8164, 8165, 8166, 8167, 8168, 8169, 8170, 8171, 8172, 8173,+ 8174, 8175, 8178, 8179, 8180, 8182, 8183, 8184, 8185, 8186, 8187, 8188, 8189, 8192, 8193, 8486,+ 8490, 8491, 8602, 8603, 8622, 8653, 8654, 8655, 8708, 8713, 8716, 8740, 8742, 8769, 8772, 8775,+ 8777, 8800, 8802, 8813, 8814, 8815, 8816, 8817, 8820, 8821, 8824, 8825, 8832, 8833, 8836, 8837,+ 8840, 8841, 8876, 8877, 8878, 8879, 8928, 8929, 8930, 8931, 8938, 8939, 8940, 8941, 9001, 9002,+ 10972, 12364, 12366, 12368, 12370, 12372, 12374, 12376, 12378, 12380, 12382, 12384, 12386,+ 12389, 12391, 12393, 12400, 12401, 12403, 12404, 12406, 12407, 12409, 12410, 12412, 12413,+ 12436, 12446, 12460, 12462, 12464, 12466, 12468, 12470, 12472, 12474, 12476, 12478, 12480,+ 12482, 12485, 12487, 12489, 12496, 12497, 12499, 12500, 12502, 12503, 12505, 12506, 12508,+ 12509, 12532, 12535, 12536, 12537, 12538, 12542, 63744, 63745, 63746, 63747, 63748, 63749,+ 63750, 63751, 63752, 63753, 63754, 63755, 63756, 63757, 63758, 63759, 63760, 63761, 63762,+ 63763, 63764, 63765, 63766, 63767, 63768, 63769, 63770, 63771, 63772, 63773, 63774, 63775,+ 63776, 63777, 63778, 63779, 63780, 63781, 63782, 63783, 63784, 63785, 63786, 63787, 63788,+ 63789, 63790, 63791, 63792, 63793, 63794, 63795, 63796, 63797, 63798, 63799, 63800, 63801,+ 63802, 63803, 63804, 63805, 63806, 63807, 63808, 63809, 63810, 63811, 63812, 63813, 63814,+ 63815, 63816, 63817, 63818, 63819, 63820, 63821, 63822, 63823, 63824, 63825, 63826, 63827,+ 63828, 63829, 63830, 63831, 63832, 63833, 63834, 63835, 63836, 63837, 63838, 63839, 63840,+ 63841, 63842, 63843, 63844, 63845, 63846, 63847, 63848, 63849, 63850, 63851, 63852, 63853,+ 63854, 63855, 63856, 63857, 63858, 63859, 63860, 63861, 63862, 63863, 63864, 63865, 63866,+ 63867, 63868, 63869, 63870, 63871, 63872, 63873, 63874, 63875, 63876, 63877, 63878, 63879,+ 63880, 63881, 63882, 63883, 63884, 63885, 63886, 63887, 63888, 63889, 63890, 63891, 63892,+ 63893, 63894, 63895, 63896, 63897, 63898, 63899, 63900, 63901, 63902, 63903, 63904, 63905,+ 63906, 63907, 63908, 63909, 63910, 63911, 63912, 63913, 63914, 63915, 63916, 63917, 63918,+ 63919, 63920, 63921, 63922, 63923, 63924, 63925, 63926, 63927, 63928, 63929, 63930, 63931,+ 63932, 63933, 63934, 63935, 63936, 63937, 63938, 63939, 63940, 63941, 63942, 63943, 63944,+ 63945, 63946, 63947, 63948, 63949, 63950, 63951, 63952, 63953, 63954, 63955, 63956, 63957,+ 63958, 63959, 63960, 63961, 63962, 63963, 63964, 63965, 63966, 63967, 63968, 63969, 63970,+ 63971, 63972, 63973, 63974, 63975, 63976, 63977, 63978, 63979, 63980, 63981, 63982, 63983,+ 63984, 63985, 63986, 63987, 63988, 63989, 63990, 63991, 63992, 63993, 63994, 63995, 63996,+ 63997, 63998, 63999, 64000, 64001, 64002, 64003, 64004, 64005, 64006, 64007, 64008, 64009,+ 64010, 64011, 64012, 64013, 64016, 64018, 64021, 64022, 64023, 64024, 64025, 64026, 64027,+ 64028, 64029, 64030, 64032, 64034, 64037, 64038, 64042, 64043, 64044, 64045, 64046, 64047,+ 64048, 64049, 64050, 64051, 64052, 64053, 64054, 64055, 64056, 64057, 64058, 64059, 64060,+ 64061, 64062, 64063, 64064, 64065, 64066, 64067, 64068, 64069, 64070, 64071, 64072, 64073,+ 64074, 64075, 64076, 64077, 64078, 64079, 64080, 64081, 64082, 64083, 64084, 64085, 64086,+ 64087, 64088, 64089, 64090, 64091, 64092, 64093, 64094, 64095, 64096, 64097, 64098, 64099,+ 64100, 64101, 64102, 64103, 64104, 64105, 64106, 64107, 64108, 64109, 64112, 64113, 64114,+ 64115, 64116, 64117, 64118, 64119, 64120, 64121, 64122, 64123, 64124, 64125, 64126, 64127,+ 64128, 64129, 64130, 64131, 64132, 64133, 64134, 64135, 64136, 64137, 64138, 64139, 64140,+ 64141, 64142, 64143, 64144, 64145, 64146, 64147, 64148, 64149, 64150, 64151, 64152, 64153,+ 64154, 64155, 64156, 64157, 64158, 64159, 64160, 64161, 64162, 64163, 64164, 64165, 64166,+ 64167, 64168, 64169, 64170, 64171, 64172, 64173, 64174, 64175, 64176, 64177, 64178, 64179,+ 64180, 64181, 64182, 64183, 64184, 64185, 64186, 64187, 64188, 64189, 64190, 64191, 64192,+ 64193, 64194, 64195, 64196, 64197, 64198, 64199, 64200, 64201, 64202, 64203, 64204, 64205,+ 64206, 64207, 64208, 64209, 64210, 64211, 64212, 64213, 64214, 64215, 64216, 64217, 64285,+ 64287, 64298, 64299, 64300, 64301, 64302, 64303, 64304, 64305, 64306, 64307, 64308, 64309,+ 64310, 64312, 64313, 64314, 64315, 64316, 64318, 64320, 64321, 64323, 64324, 64326, 64327,+ 64328, 64329, 64330, 64331, 64332, 64333, 64334, 69786, 69788, 69803, 69934, 69935, 70475,+ 70476, 70843, 70844, 70846, 71098, 71099, 119134, 119135, 119136, 119137, 119138, 119139,+ 119140, 119227, 119228, 119229, 119230, 119231, 119232, 194560, 194561, 194562, 194563, 194564,+ 194565, 194566, 194567, 194568, 194569, 194570, 194571, 194572, 194573, 194574, 194575, 194576,+ 194577, 194578, 194579, 194580, 194581, 194582, 194583, 194584, 194585, 194586, 194587, 194588,+ 194589, 194590, 194591, 194592, 194593, 194594, 194595, 194596, 194597, 194598, 194599, 194600,+ 194601, 194602, 194603, 194604, 194605, 194606, 194607, 194608, 194609, 194610, 194611, 194612,+ 194613, 194614, 194615, 194616, 194617, 194618, 194619, 194620, 194621, 194622, 194623, 194624,+ 194625, 194626, 194627, 194628, 194629, 194630, 194631, 194632, 194633, 194634, 194635, 194636,+ 194637, 194638, 194639, 194640, 194641, 194642, 194643, 194644, 194645, 194646, 194647, 194648,+ 194649, 194650, 194651, 194652, 194653, 194654, 194655, 194656, 194657, 194658, 194659, 194660,+ 194661, 194662, 194663, 194664, 194665, 194666, 194667, 194668, 194669, 194670, 194671, 194672,+ 194673, 194674, 194675, 194676, 194677, 194678, 194679, 194680, 194681, 194682, 194683, 194684,+ 194685, 194686, 194687, 194688, 194689, 194690, 194691, 194692, 194693, 194694, 194695, 194696,+ 194697, 194698, 194699, 194700, 194701, 194702, 194703, 194704, 194705, 194706, 194707, 194708,+ 194709, 194710, 194711, 194712, 194713, 194714, 194715, 194716, 194717, 194718, 194719, 194720,+ 194721, 194722, 194723, 194724, 194725, 194726, 194727, 194728, 194729, 194730, 194731, 194732,+ 194733, 194734, 194735, 194736, 194737, 194738, 194739, 194740, 194741, 194742, 194743, 194744,+ 194745, 194746, 194747, 194748, 194749, 194750, 194751, 194752, 194753, 194754, 194755, 194756,+ 194757, 194758, 194759, 194760, 194761, 194762, 194763, 194764, 194765, 194766, 194767, 194768,+ 194769, 194770, 194771, 194772, 194773, 194774, 194775, 194776, 194777, 194778, 194779, 194780,+ 194781, 194782, 194783, 194784, 194785, 194786, 194787, 194788, 194789, 194790, 194791, 194792,+ 194793, 194794, 194795, 194796, 194797, 194798, 194799, 194800, 194801, 194802, 194803, 194804,+ 194805, 194806, 194807, 194808, 194809, 194810, 194811, 194812, 194813, 194814, 194815, 194816,+ 194817, 194818, 194819, 194820, 194821, 194822, 194823, 194824, 194825, 194826, 194827, 194828,+ 194829, 194830, 194831, 194832, 194833, 194834, 194835, 194836, 194837, 194838, 194839, 194840,+ 194841, 194842, 194843, 194844, 194845, 194846, 194847, 194848, 194849, 194850, 194851, 194852,+ 194853, 194854, 194855, 194856, 194857, 194858, 194859, 194860, 194861, 194862, 194863, 194864,+ 194865, 194866, 194867, 194868, 194869, 194870, 194871, 194872, 194873, 194874, 194875, 194876,+ 194877, 194878, 194879, 194880, 194881, 194882, 194883, 194884, 194885, 194886, 194887, 194888,+ 194889, 194890, 194891, 194892, 194893, 194894, 194895, 194896, 194897, 194898, 194899, 194900,+ 194901, 194902, 194903, 194904, 194905, 194906, 194907, 194908, 194909, 194910, 194911, 194912,+ 194913, 194914, 194915, 194916, 194917, 194918, 194919, 194920, 194921, 194922, 194923, 194924,+ 194925, 194926, 194927, 194928, 194929, 194930, 194931, 194932, 194933, 194934, 194935, 194936,+ 194937, 194938, 194939, 194940, 194941, 194942, 194943, 194944, 194945, 194946, 194947, 194948,+ 194949, 194950, 194951, 194952, 194953, 194954, 194955, 194956, 194957, 194958, 194959, 194960,+ 194961, 194962, 194963, 194964, 194965, 194966, 194967, 194968, 194969, 194970, 194971, 194972,+ 194973, 194974, 194975, 194976, 194977, 194978, 194979, 194980, 194981, 194982, 194983, 194984,+ 194985, 194986, 194987, 194988, 194989, 194990, 194991, 194992, 194993, 194994, 194995, 194996,+ 194997, 194998, 194999, 195000, 195001, 195002, 195003, 195004, 195005, 195006, 195007, 195008,+ 195009, 195010, 195011, 195012, 195013, 195014, 195015, 195016, 195017, 195018, 195019, 195020,+ 195021, 195022, 195023, 195024, 195025, 195026, 195027, 195028, 195029, 195030, 195031, 195032,+ 195033, 195034, 195035, 195036, 195037, 195038, 195039, 195040, 195041, 195042, 195043, 195044,+ 195045, 195046, 195047, 195048, 195049, 195050, 195051, 195052, 195053, 195054, 195055, 195056,+ 195057, 195058, 195059, 195060, 195061, 195062, 195063, 195064, 195065, 195066, 195067, 195068,+ 195069, 195070, 195071, 195072, 195073, 195074, 195075, 195076, 195077, 195078, 195079, 195080,+ 195081, 195082, 195083, 195084, 195085, 195086, 195087, 195088, 195089, 195090, 195091, 195092,+ 195093, 195094, 195095, 195096, 195097, 195098, 195099, 195100, 195101,+};++// offsets into k_canon_pool+inline constexpr uint32_t k_canon_offsets[] = {+ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48,+ 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96,+ 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134,+ 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 160, 162, 164, 166, 168, 170, 172,+ 174, 176, 178, 180, 182, 184, 186, 188, 190, 192, 194, 196, 198, 200, 202, 204, 206, 208, 210,+ 212, 214, 216, 218, 220, 222, 224, 226, 228, 230, 232, 234, 236, 238, 240, 242, 244, 246, 248,+ 250, 252, 254, 256, 258, 260, 262, 264, 266, 268, 270, 272, 274, 276, 278, 280, 282, 284, 286,+ 288, 290, 292, 294, 296, 298, 300, 302, 304, 306, 308, 310, 312, 314, 316, 318, 320, 322, 324,+ 326, 328, 330, 332, 334, 336, 338, 340, 342, 344, 346, 349, 352, 355, 358, 361, 364, 367, 370,+ 373, 376, 379, 382, 384, 386, 388, 390, 392, 394, 396, 398, 401, 404, 406, 408, 410, 412, 414,+ 416, 418, 421, 424, 426, 428, 430, 432, 434, 436, 438, 440, 442, 444, 446, 448, 450, 452, 454,+ 456, 458, 460, 462, 464, 466, 468, 470, 472, 474, 476, 478, 480, 482, 484, 486, 488, 490, 492,+ 494, 496, 498, 500, 503, 506, 509, 512, 514, 516, 519, 522, 524, 526, 527, 528, 529, 531, 532,+ 533, 535, 537, 538, 540, 542, 544, 546, 548, 550, 553, 555, 557, 559, 561, 563, 565, 568, 570,+ 572, 574, 576, 578, 580, 582, 584, 586, 588, 590, 592, 594, 596, 598, 600, 602, 604, 606, 608,+ 610, 612, 614, 616, 618, 620, 622, 624, 626, 628, 630, 632, 634, 636, 638, 640, 642, 644, 646,+ 648, 650, 652, 654, 656, 658, 660, 662, 664, 666, 668, 670, 672, 674, 676, 678, 680, 682, 684,+ 686, 688, 690, 692, 694, 696, 698, 700, 702, 704, 706, 708, 710, 712, 714, 716, 718, 720, 722,+ 724, 726, 728, 730, 732, 734, 736, 738, 740, 742, 744, 746, 748, 750, 752, 754, 756, 758, 760,+ 762, 764, 766, 768, 770, 772, 774, 777, 779, 781, 783, 785, 787, 790, 792, 794, 796, 798, 800,+ 802, 804, 806, 808, 810, 812, 814, 816, 818, 820, 822, 824, 826, 828, 830, 832, 834, 836, 838,+ 840, 842, 844, 846, 848, 850, 852, 854, 856, 858, 860, 862, 864, 866, 869, 872, 874, 876, 878,+ 880, 882, 884, 886, 888, 890, 892, 895, 898, 901, 904, 906, 908, 910, 912, 915, 918, 920, 922,+ 924, 926, 928, 930, 932, 934, 936, 938, 940, 942, 944, 946, 948, 950, 953, 956, 958, 960, 962,+ 964, 966, 968, 970, 972, 975, 978, 980, 982, 984, 986, 988, 990, 992, 994, 996, 998, 1000,+ 1002, 1004, 1006, 1008, 1010, 1012, 1014, 1017, 1020, 1023, 1026, 1029, 1032, 1035, 1038, 1040,+ 1042, 1044, 1046, 1048, 1050, 1052, 1054, 1057, 1060, 1062, 1064, 1066, 1068, 1070, 1072, 1075,+ 1078, 1081, 1084, 1087, 1090, 1092, 1094, 1096, 1098, 1100, 1102, 1104, 1106, 1108, 1110, 1112,+ 1114, 1116, 1118, 1121, 1124, 1127, 1130, 1132, 1134, 1136, 1138, 1140, 1142, 1144, 1146, 1148,+ 1150, 1152, 1154, 1156, 1158, 1160, 1162, 1164, 1166, 1168, 1170, 1172, 1174, 1176, 1178, 1180,+ 1182, 1184, 1186, 1188, 1190, 1192, 1194, 1196, 1198, 1200, 1203, 1206, 1209, 1212, 1215, 1218,+ 1221, 1224, 1227, 1230, 1233, 1236, 1239, 1242, 1245, 1248, 1251, 1254, 1257, 1260, 1262, 1264,+ 1266, 1268, 1270, 1272, 1275, 1278, 1281, 1284, 1287, 1290, 1293, 1296, 1299, 1302, 1304, 1306,+ 1308, 1310, 1312, 1314, 1316, 1318, 1321, 1324, 1327, 1330, 1333, 1336, 1339, 1342, 1345, 1348,+ 1351, 1354, 1357, 1360, 1363, 1366, 1369, 1372, 1375, 1378, 1380, 1382, 1384, 1386, 1389, 1392,+ 1395, 1398, 1401, 1404, 1407, 1410, 1413, 1416, 1418, 1420, 1422, 1424, 1426, 1428, 1430, 1432,+ 1434, 1436, 1439, 1442, 1445, 1448, 1451, 1454, 1456, 1458, 1461, 1464, 1467, 1470, 1473, 1476,+ 1478, 1480, 1483, 1486, 1489, 1492, 1494, 1496, 1499, 1502, 1505, 1508, 1510, 1512, 1515, 1518,+ 1521, 1524, 1527, 1530, 1532, 1534, 1537, 1540, 1543, 1546, 1549, 1552, 1554, 1556, 1559, 1562,+ 1565, 1568, 1571, 1574, 1576, 1578, 1581, 1584, 1587, 1590, 1593, 1596, 1598, 1600, 1603, 1606,+ 1609, 1612, 1614, 1616, 1619, 1622, 1625, 1628, 1630, 1632, 1635, 1638, 1641, 1644, 1647, 1650,+ 1652, 1655, 1658, 1661, 1663, 1665, 1668, 1671, 1674, 1677, 1680, 1683, 1685, 1687, 1690, 1693,+ 1696, 1699, 1702, 1705, 1707, 1709, 1711, 1713, 1715, 1717, 1719, 1721, 1723, 1725, 1727, 1729,+ 1731, 1733, 1736, 1739, 1743, 1747, 1751, 1755, 1759, 1763, 1766, 1769, 1773, 1777, 1781, 1785,+ 1789, 1793, 1796, 1799, 1803, 1807, 1811, 1815, 1819, 1823, 1826, 1829, 1833, 1837, 1841, 1845,+ 1849, 1853, 1856, 1859, 1863, 1867, 1871, 1875, 1879, 1883, 1886, 1889, 1893, 1897, 1901, 1905,+ 1909, 1913, 1915, 1917, 1920, 1922, 1925, 1927, 1930, 1932, 1934, 1936, 1938, 1940, 1941, 1943,+ 1946, 1948, 1951, 1953, 1956, 1958, 1960, 1962, 1964, 1966, 1968, 1970, 1972, 1974, 1976, 1979,+ 1982, 1984, 1987, 1989, 1991, 1993, 1995, 1997, 1999, 2001, 2003, 2005, 2008, 2011, 2013, 2015,+ 2017, 2020, 2022, 2024, 2026, 2028, 2030, 2032, 2034, 2035, 2038, 2040, 2043, 2045, 2048, 2050,+ 2052, 2054, 2056, 2058, 2059, 2060, 2061, 2062, 2063, 2065, 2067, 2069, 2071, 2073, 2075, 2077,+ 2079, 2081, 2083, 2085, 2087, 2089, 2091, 2093, 2095, 2097, 2099, 2101, 2103, 2105, 2107, 2109,+ 2111, 2113, 2115, 2117, 2119, 2121, 2123, 2125, 2127, 2129, 2131, 2133, 2135, 2137, 2139, 2141,+ 2143, 2145, 2147, 2149, 2151, 2153, 2154, 2155, 2157, 2159, 2161, 2163, 2165, 2167, 2169, 2171,+ 2173, 2175, 2177, 2179, 2181, 2183, 2185, 2187, 2189, 2191, 2193, 2195, 2197, 2199, 2201, 2203,+ 2205, 2207, 2209, 2211, 2213, 2215, 2217, 2219, 2221, 2223, 2225, 2227, 2229, 2231, 2233, 2235,+ 2237, 2239, 2241, 2243, 2245, 2247, 2249, 2251, 2253, 2255, 2257, 2259, 2261, 2263, 2265, 2267,+ 2269, 2271, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286,+ 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302,+ 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318,+ 2319, 2320, 2321, 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334,+ 2335, 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350,+ 2351, 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366,+ 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382,+ 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398,+ 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414,+ 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2429, 2430,+ 2431, 2432, 2433, 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2446,+ 2447, 2448, 2449, 2450, 2451, 2452, 2453, 2454, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462,+ 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2476, 2477, 2478,+ 2479, 2480, 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494,+ 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510,+ 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526,+ 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542,+ 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558,+ 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2568, 2569, 2570, 2571, 2572, 2573, 2574,+ 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590,+ 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606,+ 2607, 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622,+ 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2638,+ 2639, 2640, 2641, 2642, 2643, 2644, 2645, 2646, 2647, 2648, 2649, 2650, 2651, 2652, 2653, 2654,+ 2655, 2656, 2657, 2658, 2659, 2660, 2661, 2662, 2663, 2664, 2665, 2666, 2667, 2668, 2669, 2670,+ 2671, 2672, 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, 2684, 2685, 2686,+ 2687, 2688, 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, 2700, 2701, 2702,+ 2703, 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718,+ 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2735,+ 2737, 2739, 2741, 2744, 2747, 2749, 2751, 2753, 2755, 2757, 2759, 2761, 2763, 2765, 2767, 2769,+ 2771, 2773, 2775, 2777, 2779, 2781, 2783, 2785, 2787, 2789, 2791, 2793, 2795, 2797, 2799, 2801,+ 2803, 2805, 2807, 2809, 2811, 2813, 2815, 2817, 2819, 2821, 2823, 2825, 2827, 2829, 2831, 2834,+ 2837, 2840, 2843, 2846, 2848, 2850, 2853, 2856, 2859, 2862, 2863, 2864, 2865, 2866, 2867, 2868,+ 2869, 2870, 2871, 2872, 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, 2884,+ 2885, 2886, 2887, 2888, 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, 2900,+ 2901, 2902, 2903, 2904, 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, 2916,+ 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931, 2932,+ 2933, 2934, 2935, 2936, 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, 2948,+ 2949, 2950, 2951, 2952, 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, 2964,+ 2965, 2966, 2967, 2968, 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, 2980,+ 2981, 2982, 2983, 2984, 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, 2996,+ 2997, 2998, 2999, 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, 3012,+ 3013, 3014, 3015, 3016, 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, 3028,+ 3029, 3030, 3031, 3032, 3033, 3034, 3035, 3036, 3037, 3038, 3039, 3040, 3041, 3042, 3043, 3044,+ 3045, 3046, 3047, 3048, 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, 3060,+ 3061, 3062, 3063, 3064, 3065, 3066, 3067, 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, 3076,+ 3077, 3078, 3079, 3080, 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, 3092,+ 3093, 3094, 3095, 3096, 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108,+ 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124,+ 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, 3140,+ 3141, 3142, 3143, 3144, 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, 3156,+ 3157, 3158, 3159, 3160, 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, 3172,+ 3173, 3174, 3175, 3176, 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, 3188,+ 3189, 3190, 3191, 3192, 3193, 3194, 3195, 3196, 3197, 3198, 3199, 3200, 3201, 3202, 3203, 3204,+ 3205, 3206, 3207, 3208, 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, 3220,+ 3221, 3222, 3223, 3224, 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, 3236,+ 3237, 3238, 3239, 3240, 3241, 3242, 3243, 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, 3252,+ 3253, 3254, 3255, 3256, 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, 3268,+ 3269, 3270, 3271, 3272, 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, 3284,+ 3285, 3286, 3287, 3288, 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, 3300,+ 3301, 3302, 3303, 3304, 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, 3316,+ 3317, 3318, 3319, 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, 3332,+ 3333, 3334, 3335, 3336, 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, 3348,+ 3349, 3350, 3351, 3352, 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, 3364,+ 3365, 3366, 3367, 3368, 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, 3380,+ 3381, 3382, 3383, 3384, 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, 3396,+ 3397, 3398, 3399, 3400, 3401, 3402, 3403, 3404,+};++// the decompositions of k_canon_keys+inline constexpr uint32_t k_canon_pool[] = {+ 65, 768, 65, 769, 65, 770, 65, 771, 65, 776, 65, 778, 67, 807, 69, 768, 69, 769, 69, 770, 69,+ 776, 73, 768, 73, 769, 73, 770, 73, 776, 78, 771, 79, 768, 79, 769, 79, 770, 79, 771, 79, 776,+ 85, 768, 85, 769, 85, 770, 85, 776, 89, 769, 97, 768, 97, 769, 97, 770, 97, 771, 97, 776, 97,+ 778, 99, 807, 101, 768, 101, 769, 101, 770, 101, 776, 105, 768, 105, 769, 105, 770, 105, 776,+ 110, 771, 111, 768, 111, 769, 111, 770, 111, 771, 111, 776, 117, 768, 117, 769, 117, 770, 117,+ 776, 121, 769, 121, 776, 65, 772, 97, 772, 65, 774, 97, 774, 65, 808, 97, 808, 67, 769, 99,+ 769, 67, 770, 99, 770, 67, 775, 99, 775, 67, 780, 99, 780, 68, 780, 100, 780, 69, 772, 101,+ 772, 69, 774, 101, 774, 69, 775, 101, 775, 69, 808, 101, 808, 69, 780, 101, 780, 71, 770, 103,+ 770, 71, 774, 103, 774, 71, 775, 103, 775, 71, 807, 103, 807, 72, 770, 104, 770, 73, 771, 105,+ 771, 73, 772, 105, 772, 73, 774, 105, 774, 73, 808, 105, 808, 73, 775, 74, 770, 106, 770, 75,+ 807, 107, 807, 76, 769, 108, 769, 76, 807, 108, 807, 76, 780, 108, 780, 78, 769, 110, 769, 78,+ 807, 110, 807, 78, 780, 110, 780, 79, 772, 111, 772, 79, 774, 111, 774, 79, 779, 111, 779, 82,+ 769, 114, 769, 82, 807, 114, 807, 82, 780, 114, 780, 83, 769, 115, 769, 83, 770, 115, 770, 83,+ 807, 115, 807, 83, 780, 115, 780, 84, 807, 116, 807, 84, 780, 116, 780, 85, 771, 117, 771, 85,+ 772, 117, 772, 85, 774, 117, 774, 85, 778, 117, 778, 85, 779, 117, 779, 85, 808, 117, 808, 87,+ 770, 119, 770, 89, 770, 121, 770, 89, 776, 90, 769, 122, 769, 90, 775, 122, 775, 90, 780, 122,+ 780, 79, 795, 111, 795, 85, 795, 117, 795, 65, 780, 97, 780, 73, 780, 105, 780, 79, 780, 111,+ 780, 85, 780, 117, 780, 85, 776, 772, 117, 776, 772, 85, 776, 769, 117, 776, 769, 85, 776, 780,+ 117, 776, 780, 85, 776, 768, 117, 776, 768, 65, 776, 772, 97, 776, 772, 65, 775, 772, 97, 775,+ 772, 198, 772, 230, 772, 71, 780, 103, 780, 75, 780, 107, 780, 79, 808, 111, 808, 79, 808, 772,+ 111, 808, 772, 439, 780, 658, 780, 106, 780, 71, 769, 103, 769, 78, 768, 110, 768, 65, 778,+ 769, 97, 778, 769, 198, 769, 230, 769, 216, 769, 248, 769, 65, 783, 97, 783, 65, 785, 97, 785,+ 69, 783, 101, 783, 69, 785, 101, 785, 73, 783, 105, 783, 73, 785, 105, 785, 79, 783, 111, 783,+ 79, 785, 111, 785, 82, 783, 114, 783, 82, 785, 114, 785, 85, 783, 117, 783, 85, 785, 117, 785,+ 83, 806, 115, 806, 84, 806, 116, 806, 72, 780, 104, 780, 65, 775, 97, 775, 69, 807, 101, 807,+ 79, 776, 772, 111, 776, 772, 79, 771, 772, 111, 771, 772, 79, 775, 111, 775, 79, 775, 772, 111,+ 775, 772, 89, 772, 121, 772, 768, 769, 787, 776, 769, 697, 59, 168, 769, 913, 769, 183, 917,+ 769, 919, 769, 921, 769, 927, 769, 933, 769, 937, 769, 953, 776, 769, 921, 776, 933, 776, 945,+ 769, 949, 769, 951, 769, 953, 769, 965, 776, 769, 953, 776, 965, 776, 959, 769, 965, 769, 969,+ 769, 978, 769, 978, 776, 1045, 768, 1045, 776, 1043, 769, 1030, 776, 1050, 769, 1048, 768,+ 1059, 774, 1048, 774, 1080, 774, 1077, 768, 1077, 776, 1075, 769, 1110, 776, 1082, 769, 1080,+ 768, 1091, 774, 1140, 783, 1141, 783, 1046, 774, 1078, 774, 1040, 774, 1072, 774, 1040, 776,+ 1072, 776, 1045, 774, 1077, 774, 1240, 776, 1241, 776, 1046, 776, 1078, 776, 1047, 776, 1079,+ 776, 1048, 772, 1080, 772, 1048, 776, 1080, 776, 1054, 776, 1086, 776, 1256, 776, 1257, 776,+ 1069, 776, 1101, 776, 1059, 772, 1091, 772, 1059, 776, 1091, 776, 1059, 779, 1091, 779, 1063,+ 776, 1095, 776, 1067, 776, 1099, 776, 1575, 1619, 1575, 1620, 1608, 1620, 1575, 1621, 1610,+ 1620, 1749, 1620, 1729, 1620, 1746, 1620, 2344, 2364, 2352, 2364, 2355, 2364, 2325, 2364, 2326,+ 2364, 2327, 2364, 2332, 2364, 2337, 2364, 2338, 2364, 2347, 2364, 2351, 2364, 2503, 2494, 2503,+ 2519, 2465, 2492, 2466, 2492, 2479, 2492, 2610, 2620, 2616, 2620, 2582, 2620, 2583, 2620, 2588,+ 2620, 2603, 2620, 2887, 2902, 2887, 2878, 2887, 2903, 2849, 2876, 2850, 2876, 2962, 3031, 3014,+ 3006, 3015, 3006, 3014, 3031, 3142, 3158, 3263, 3285, 3270, 3285, 3270, 3286, 3270, 3266, 3270,+ 3266, 3285, 3398, 3390, 3399, 3390, 3398, 3415, 3545, 3530, 3545, 3535, 3545, 3535, 3530, 3545,+ 3551, 3906, 4023, 3916, 4023, 3921, 4023, 3926, 4023, 3931, 4023, 3904, 4021, 3953, 3954, 3953,+ 3956, 4018, 3968, 4019, 3968, 3953, 3968, 3986, 4023, 3996, 4023, 4001, 4023, 4006, 4023, 4011,+ 4023, 3984, 4021, 4133, 4142, 6917, 6965, 6919, 6965, 6921, 6965, 6923, 6965, 6925, 6965, 6929,+ 6965, 6970, 6965, 6972, 6965, 6974, 6965, 6975, 6965, 6978, 6965, 65, 805, 97, 805, 66, 775,+ 98, 775, 66, 803, 98, 803, 66, 817, 98, 817, 67, 807, 769, 99, 807, 769, 68, 775, 100, 775, 68,+ 803, 100, 803, 68, 817, 100, 817, 68, 807, 100, 807, 68, 813, 100, 813, 69, 772, 768, 101, 772,+ 768, 69, 772, 769, 101, 772, 769, 69, 813, 101, 813, 69, 816, 101, 816, 69, 807, 774, 101, 807,+ 774, 70, 775, 102, 775, 71, 772, 103, 772, 72, 775, 104, 775, 72, 803, 104, 803, 72, 776, 104,+ 776, 72, 807, 104, 807, 72, 814, 104, 814, 73, 816, 105, 816, 73, 776, 769, 105, 776, 769, 75,+ 769, 107, 769, 75, 803, 107, 803, 75, 817, 107, 817, 76, 803, 108, 803, 76, 803, 772, 108, 803,+ 772, 76, 817, 108, 817, 76, 813, 108, 813, 77, 769, 109, 769, 77, 775, 109, 775, 77, 803, 109,+ 803, 78, 775, 110, 775, 78, 803, 110, 803, 78, 817, 110, 817, 78, 813, 110, 813, 79, 771, 769,+ 111, 771, 769, 79, 771, 776, 111, 771, 776, 79, 772, 768, 111, 772, 768, 79, 772, 769, 111,+ 772, 769, 80, 769, 112, 769, 80, 775, 112, 775, 82, 775, 114, 775, 82, 803, 114, 803, 82, 803,+ 772, 114, 803, 772, 82, 817, 114, 817, 83, 775, 115, 775, 83, 803, 115, 803, 83, 769, 775, 115,+ 769, 775, 83, 780, 775, 115, 780, 775, 83, 803, 775, 115, 803, 775, 84, 775, 116, 775, 84, 803,+ 116, 803, 84, 817, 116, 817, 84, 813, 116, 813, 85, 804, 117, 804, 85, 816, 117, 816, 85, 813,+ 117, 813, 85, 771, 769, 117, 771, 769, 85, 772, 776, 117, 772, 776, 86, 771, 118, 771, 86, 803,+ 118, 803, 87, 768, 119, 768, 87, 769, 119, 769, 87, 776, 119, 776, 87, 775, 119, 775, 87, 803,+ 119, 803, 88, 775, 120, 775, 88, 776, 120, 776, 89, 775, 121, 775, 90, 770, 122, 770, 90, 803,+ 122, 803, 90, 817, 122, 817, 104, 817, 116, 776, 119, 778, 121, 778, 383, 775, 65, 803, 97,+ 803, 65, 777, 97, 777, 65, 770, 769, 97, 770, 769, 65, 770, 768, 97, 770, 768, 65, 770, 777,+ 97, 770, 777, 65, 770, 771, 97, 770, 771, 65, 803, 770, 97, 803, 770, 65, 774, 769, 97, 774,+ 769, 65, 774, 768, 97, 774, 768, 65, 774, 777, 97, 774, 777, 65, 774, 771, 97, 774, 771, 65,+ 803, 774, 97, 803, 774, 69, 803, 101, 803, 69, 777, 101, 777, 69, 771, 101, 771, 69, 770, 769,+ 101, 770, 769, 69, 770, 768, 101, 770, 768, 69, 770, 777, 101, 770, 777, 69, 770, 771, 101,+ 770, 771, 69, 803, 770, 101, 803, 770, 73, 777, 105, 777, 73, 803, 105, 803, 79, 803, 111, 803,+ 79, 777, 111, 777, 79, 770, 769, 111, 770, 769, 79, 770, 768, 111, 770, 768, 79, 770, 777, 111,+ 770, 777, 79, 770, 771, 111, 770, 771, 79, 803, 770, 111, 803, 770, 79, 795, 769, 111, 795,+ 769, 79, 795, 768, 111, 795, 768, 79, 795, 777, 111, 795, 777, 79, 795, 771, 111, 795, 771, 79,+ 795, 803, 111, 795, 803, 85, 803, 117, 803, 85, 777, 117, 777, 85, 795, 769, 117, 795, 769, 85,+ 795, 768, 117, 795, 768, 85, 795, 777, 117, 795, 777, 85, 795, 771, 117, 795, 771, 85, 795,+ 803, 117, 795, 803, 89, 768, 121, 768, 89, 803, 121, 803, 89, 777, 121, 777, 89, 771, 121, 771,+ 945, 787, 945, 788, 945, 787, 768, 945, 788, 768, 945, 787, 769, 945, 788, 769, 945, 787, 834,+ 945, 788, 834, 913, 787, 913, 788, 913, 787, 768, 913, 788, 768, 913, 787, 769, 913, 788, 769,+ 913, 787, 834, 913, 788, 834, 949, 787, 949, 788, 949, 787, 768, 949, 788, 768, 949, 787, 769,+ 949, 788, 769, 917, 787, 917, 788, 917, 787, 768, 917, 788, 768, 917, 787, 769, 917, 788, 769,+ 951, 787, 951, 788, 951, 787, 768, 951, 788, 768, 951, 787, 769, 951, 788, 769, 951, 787, 834,+ 951, 788, 834, 919, 787, 919, 788, 919, 787, 768, 919, 788, 768, 919, 787, 769, 919, 788, 769,+ 919, 787, 834, 919, 788, 834, 953, 787, 953, 788, 953, 787, 768, 953, 788, 768, 953, 787, 769,+ 953, 788, 769, 953, 787, 834, 953, 788, 834, 921, 787, 921, 788, 921, 787, 768, 921, 788, 768,+ 921, 787, 769, 921, 788, 769, 921, 787, 834, 921, 788, 834, 959, 787, 959, 788, 959, 787, 768,+ 959, 788, 768, 959, 787, 769, 959, 788, 769, 927, 787, 927, 788, 927, 787, 768, 927, 788, 768,+ 927, 787, 769, 927, 788, 769, 965, 787, 965, 788, 965, 787, 768, 965, 788, 768, 965, 787, 769,+ 965, 788, 769, 965, 787, 834, 965, 788, 834, 933, 788, 933, 788, 768, 933, 788, 769, 933, 788,+ 834, 969, 787, 969, 788, 969, 787, 768, 969, 788, 768, 969, 787, 769, 969, 788, 769, 969, 787,+ 834, 969, 788, 834, 937, 787, 937, 788, 937, 787, 768, 937, 788, 768, 937, 787, 769, 937, 788,+ 769, 937, 787, 834, 937, 788, 834, 945, 768, 945, 769, 949, 768, 949, 769, 951, 768, 951, 769,+ 953, 768, 953, 769, 959, 768, 959, 769, 965, 768, 965, 769, 969, 768, 969, 769, 945, 787, 837,+ 945, 788, 837, 945, 787, 768, 837, 945, 788, 768, 837, 945, 787, 769, 837, 945, 788, 769, 837,+ 945, 787, 834, 837, 945, 788, 834, 837, 913, 787, 837, 913, 788, 837, 913, 787, 768, 837, 913,+ 788, 768, 837, 913, 787, 769, 837, 913, 788, 769, 837, 913, 787, 834, 837, 913, 788, 834, 837,+ 951, 787, 837, 951, 788, 837, 951, 787, 768, 837, 951, 788, 768, 837, 951, 787, 769, 837, 951,+ 788, 769, 837, 951, 787, 834, 837, 951, 788, 834, 837, 919, 787, 837, 919, 788, 837, 919, 787,+ 768, 837, 919, 788, 768, 837, 919, 787, 769, 837, 919, 788, 769, 837, 919, 787, 834, 837, 919,+ 788, 834, 837, 969, 787, 837, 969, 788, 837, 969, 787, 768, 837, 969, 788, 768, 837, 969, 787,+ 769, 837, 969, 788, 769, 837, 969, 787, 834, 837, 969, 788, 834, 837, 937, 787, 837, 937, 788,+ 837, 937, 787, 768, 837, 937, 788, 768, 837, 937, 787, 769, 837, 937, 788, 769, 837, 937, 787,+ 834, 837, 937, 788, 834, 837, 945, 774, 945, 772, 945, 768, 837, 945, 837, 945, 769, 837, 945,+ 834, 945, 834, 837, 913, 774, 913, 772, 913, 768, 913, 769, 913, 837, 953, 168, 834, 951, 768,+ 837, 951, 837, 951, 769, 837, 951, 834, 951, 834, 837, 917, 768, 917, 769, 919, 768, 919, 769,+ 919, 837, 8127, 768, 8127, 769, 8127, 834, 953, 774, 953, 772, 953, 776, 768, 953, 776, 769,+ 953, 834, 953, 776, 834, 921, 774, 921, 772, 921, 768, 921, 769, 8190, 768, 8190, 769, 8190,+ 834, 965, 774, 965, 772, 965, 776, 768, 965, 776, 769, 961, 787, 961, 788, 965, 834, 965, 776,+ 834, 933, 774, 933, 772, 933, 768, 933, 769, 929, 788, 168, 768, 168, 769, 96, 969, 768, 837,+ 969, 837, 969, 769, 837, 969, 834, 969, 834, 837, 927, 768, 927, 769, 937, 768, 937, 769, 937,+ 837, 180, 8194, 8195, 937, 75, 65, 778, 8592, 824, 8594, 824, 8596, 824, 8656, 824, 8660, 824,+ 8658, 824, 8707, 824, 8712, 824, 8715, 824, 8739, 824, 8741, 824, 8764, 824, 8771, 824, 8773,+ 824, 8776, 824, 61, 824, 8801, 824, 8781, 824, 60, 824, 62, 824, 8804, 824, 8805, 824, 8818,+ 824, 8819, 824, 8822, 824, 8823, 824, 8826, 824, 8827, 824, 8834, 824, 8835, 824, 8838, 824,+ 8839, 824, 8866, 824, 8872, 824, 8873, 824, 8875, 824, 8828, 824, 8829, 824, 8849, 824, 8850,+ 824, 8882, 824, 8883, 824, 8884, 824, 8885, 824, 12296, 12297, 10973, 824, 12363, 12441, 12365,+ 12441, 12367, 12441, 12369, 12441, 12371, 12441, 12373, 12441, 12375, 12441, 12377, 12441,+ 12379, 12441, 12381, 12441, 12383, 12441, 12385, 12441, 12388, 12441, 12390, 12441, 12392,+ 12441, 12399, 12441, 12399, 12442, 12402, 12441, 12402, 12442, 12405, 12441, 12405, 12442,+ 12408, 12441, 12408, 12442, 12411, 12441, 12411, 12442, 12358, 12441, 12445, 12441, 12459,+ 12441, 12461, 12441, 12463, 12441, 12465, 12441, 12467, 12441, 12469, 12441, 12471, 12441,+ 12473, 12441, 12475, 12441, 12477, 12441, 12479, 12441, 12481, 12441, 12484, 12441, 12486,+ 12441, 12488, 12441, 12495, 12441, 12495, 12442, 12498, 12441, 12498, 12442, 12501, 12441,+ 12501, 12442, 12504, 12441, 12504, 12442, 12507, 12441, 12507, 12442, 12454, 12441, 12527,+ 12441, 12528, 12441, 12529, 12441, 12530, 12441, 12541, 12441, 35912, 26356, 36554, 36040,+ 28369, 20018, 21477, 40860, 40860, 22865, 37329, 21895, 22856, 25078, 30313, 32645, 34367,+ 34746, 35064, 37007, 27138, 27931, 28889, 29662, 33853, 37226, 39409, 20098, 21365, 27396,+ 29211, 34349, 40478, 23888, 28651, 34253, 35172, 25289, 33240, 34847, 24266, 26391, 28010,+ 29436, 37070, 20358, 20919, 21214, 25796, 27347, 29200, 30439, 32769, 34310, 34396, 36335,+ 38706, 39791, 40442, 30860, 31103, 32160, 33737, 37636, 40575, 35542, 22751, 24324, 31840,+ 32894, 29282, 30922, 36034, 38647, 22744, 23650, 27155, 28122, 28431, 32047, 32311, 38475,+ 21202, 32907, 20956, 20940, 31260, 32190, 33777, 38517, 35712, 25295, 27138, 35582, 20025,+ 23527, 24594, 29575, 30064, 21271, 30971, 20415, 24489, 19981, 27852, 25976, 32034, 21443,+ 22622, 30465, 33865, 35498, 27578, 36784, 27784, 25342, 33509, 25504, 30053, 20142, 20841,+ 20937, 26753, 31975, 33391, 35538, 37327, 21237, 21570, 22899, 24300, 26053, 28670, 31018,+ 38317, 39530, 40599, 40654, 21147, 26310, 27511, 36706, 24180, 24976, 25088, 25754, 28451,+ 29001, 29833, 31178, 32244, 32879, 36646, 34030, 36899, 37706, 21015, 21155, 21693, 28872,+ 35010, 35498, 24265, 24565, 25467, 27566, 31806, 29557, 20196, 22265, 23527, 23994, 24604,+ 29618, 29801, 32666, 32838, 37428, 38646, 38728, 38936, 20363, 31150, 37300, 38584, 24801,+ 20102, 20698, 23534, 23615, 26009, 27138, 29134, 30274, 34044, 36988, 40845, 26248, 38446,+ 21129, 26491, 26611, 27969, 28316, 29705, 30041, 30827, 32016, 39006, 20845, 25134, 38520,+ 20523, 23833, 28138, 36650, 24459, 24900, 26647, 29575, 38534, 21033, 21519, 23653, 26131,+ 26446, 26792, 27877, 29702, 30178, 32633, 35023, 35041, 37324, 38626, 21311, 28346, 21533,+ 29136, 29848, 34298, 38563, 40023, 40607, 26519, 28107, 33256, 31435, 31520, 31890, 29376,+ 28825, 35672, 20160, 33590, 21050, 20999, 24230, 25299, 31958, 23429, 27934, 26292, 36667,+ 34892, 38477, 35211, 24275, 20800, 21952, 22618, 26228, 20958, 29482, 30410, 31036, 31070,+ 31077, 31119, 38742, 31934, 32701, 34322, 35576, 36920, 37117, 39151, 39164, 39208, 40372,+ 37086, 38583, 20398, 20711, 20813, 21193, 21220, 21329, 21917, 22022, 22120, 22592, 22696,+ 23652, 23662, 24724, 24936, 24974, 25074, 25935, 26082, 26257, 26757, 28023, 28186, 28450,+ 29038, 29227, 29730, 30865, 31038, 31049, 31048, 31056, 31062, 31069, 31117, 31118, 31296,+ 31361, 31680, 32244, 32265, 32321, 32626, 32773, 33261, 33401, 33401, 33879, 35088, 35222,+ 35585, 35641, 36051, 36104, 36790, 36920, 38627, 38911, 38971, 24693, 148206, 33304, 20006,+ 20917, 20840, 20352, 20805, 20864, 21191, 21242, 21917, 21845, 21913, 21986, 22618, 22707,+ 22852, 22868, 23138, 23336, 24274, 24281, 24425, 24493, 24792, 24910, 24840, 24974, 24928,+ 25074, 25140, 25540, 25628, 25682, 25942, 26228, 26391, 26395, 26454, 27513, 27578, 27969,+ 28379, 28363, 28450, 28702, 29038, 30631, 29237, 29359, 29482, 29809, 29958, 30011, 30237,+ 30239, 30410, 30427, 30452, 30538, 30528, 30924, 31409, 31680, 31867, 32091, 32244, 32574,+ 32773, 33618, 33775, 34681, 35137, 35206, 35222, 35519, 35576, 35531, 35585, 35582, 35565,+ 35641, 35722, 36104, 36664, 36978, 37273, 37494, 38524, 38627, 38742, 38875, 38911, 38923,+ 38971, 39698, 40860, 141386, 141380, 144341, 15261, 16408, 16441, 152137, 154832, 163539,+ 40771, 40846, 1497, 1460, 1522, 1463, 1513, 1473, 1513, 1474, 1513, 1468, 1473, 1513, 1468,+ 1474, 1488, 1463, 1488, 1464, 1488, 1468, 1489, 1468, 1490, 1468, 1491, 1468, 1492, 1468, 1493,+ 1468, 1494, 1468, 1496, 1468, 1497, 1468, 1498, 1468, 1499, 1468, 1500, 1468, 1502, 1468, 1504,+ 1468, 1505, 1468, 1507, 1468, 1508, 1468, 1510, 1468, 1511, 1468, 1512, 1468, 1513, 1468, 1514,+ 1468, 1493, 1465, 1489, 1471, 1499, 1471, 1508, 1471, 69785, 69818, 69787, 69818, 69797, 69818,+ 69937, 69927, 69938, 69927, 70471, 70462, 70471, 70487, 70841, 70842, 70841, 70832, 70841,+ 70845, 71096, 71087, 71097, 71087, 119127, 119141, 119128, 119141, 119128, 119141, 119150,+ 119128, 119141, 119151, 119128, 119141, 119152, 119128, 119141, 119153, 119128, 119141, 119154,+ 119225, 119141, 119226, 119141, 119225, 119141, 119150, 119226, 119141, 119150, 119225, 119141,+ 119151, 119226, 119141, 119151, 20029, 20024, 20033, 131362, 20320, 20398, 20411, 20482, 20602,+ 20633, 20711, 20687, 13470, 132666, 20813, 20820, 20836, 20855, 132380, 13497, 20839, 20877,+ 132427, 20887, 20900, 20172, 20908, 20917, 168415, 20981, 20995, 13535, 21051, 21062, 21106,+ 21111, 13589, 21191, 21193, 21220, 21242, 21253, 21254, 21271, 21321, 21329, 21338, 21363,+ 21373, 21375, 21375, 21375, 133676, 28784, 21450, 21471, 133987, 21483, 21489, 21510, 21662,+ 21560, 21576, 21608, 21666, 21750, 21776, 21843, 21859, 21892, 21892, 21913, 21931, 21939,+ 21954, 22294, 22022, 22295, 22097, 22132, 20999, 22766, 22478, 22516, 22541, 22411, 22578,+ 22577, 22700, 136420, 22770, 22775, 22790, 22810, 22818, 22882, 136872, 136938, 23020, 23067,+ 23079, 23000, 23142, 14062, 14076, 23304, 23358, 23358, 137672, 23491, 23512, 23527, 23539,+ 138008, 23551, 23558, 24403, 23586, 14209, 23648, 23662, 23744, 23693, 138724, 23875, 138726,+ 23918, 23915, 23932, 24033, 24034, 14383, 24061, 24104, 24125, 24169, 14434, 139651, 14460,+ 24240, 24243, 24246, 24266, 172946, 24318, 140081, 140081, 33281, 24354, 24354, 14535, 144056,+ 156122, 24418, 24427, 14563, 24474, 24525, 24535, 24569, 24705, 14650, 14620, 24724, 141012,+ 24775, 24904, 24908, 24910, 24908, 24954, 24974, 25010, 24996, 25007, 25054, 25074, 25078,+ 25104, 25115, 25181, 25265, 25300, 25424, 142092, 25405, 25340, 25448, 25475, 25572, 142321,+ 25634, 25541, 25513, 14894, 25705, 25726, 25757, 25719, 14956, 25935, 25964, 143370, 26083,+ 26360, 26185, 15129, 26257, 15112, 15076, 20882, 20885, 26368, 26268, 32941, 17369, 26391,+ 26395, 26401, 26462, 26451, 144323, 15177, 26618, 26501, 26706, 26757, 144493, 26766, 26655,+ 26900, 15261, 26946, 27043, 27114, 27304, 145059, 27355, 15384, 27425, 145575, 27476, 15438,+ 27506, 27551, 27578, 27579, 146061, 138507, 146170, 27726, 146620, 27839, 27853, 27751, 27926,+ 27966, 28023, 27969, 28009, 28024, 28037, 146718, 27956, 28207, 28270, 15667, 28363, 28359,+ 147153, 28153, 28526, 147294, 147342, 28614, 28729, 28702, 28699, 15766, 28746, 28797, 28791,+ 28845, 132389, 28997, 148067, 29084, 148395, 29224, 29237, 29264, 149000, 29312, 29333, 149301,+ 149524, 29562, 29579, 16044, 29605, 16056, 16056, 29767, 29788, 29809, 29829, 29898, 16155,+ 29988, 150582, 30014, 150674, 30064, 139679, 30224, 151457, 151480, 151620, 16380, 16392,+ 30452, 151795, 151794, 151833, 151859, 30494, 30495, 30495, 30538, 16441, 30603, 16454, 16534,+ 152605, 30798, 30860, 30924, 16611, 153126, 31062, 153242, 153285, 31119, 31211, 16687, 31296,+ 31306, 31311, 153980, 154279, 154279, 31470, 16898, 154539, 31686, 31689, 16935, 154752, 31954,+ 17056, 31976, 31971, 32000, 155526, 32099, 17153, 32199, 32258, 32325, 17204, 156200, 156231,+ 17241, 156377, 32634, 156478, 32661, 32762, 32773, 156890, 156963, 32864, 157096, 32880,+ 144223, 17365, 32946, 33027, 17419, 33086, 23221, 157607, 157621, 144275, 144284, 33281, 33284,+ 36766, 17515, 33425, 33419, 33437, 21171, 33457, 33459, 33469, 33510, 158524, 33509, 33565,+ 33635, 33709, 33571, 33725, 33767, 33879, 33619, 33738, 33740, 33756, 158774, 159083, 158933,+ 17707, 34033, 34035, 34070, 160714, 34148, 159532, 17757, 17761, 159665, 159954, 17771, 34384,+ 34396, 34407, 34409, 34473, 34440, 34574, 34530, 34681, 34600, 34667, 34694, 17879, 34785,+ 34817, 17913, 34912, 34915, 161383, 35031, 35038, 17973, 35066, 13499, 161966, 162150, 18110,+ 18119, 35488, 35565, 35722, 35925, 162984, 36011, 36033, 36123, 36215, 163631, 133124, 36299,+ 36284, 36336, 133342, 36564, 36664, 165330, 165357, 37012, 37105, 37137, 165678, 37147, 37432,+ 37591, 37592, 37500, 37881, 37909, 166906, 38283, 18837, 38327, 167287, 18918, 38595, 23986,+ 38691, 168261, 168474, 19054, 19062, 38880, 168970, 19122, 169110, 38923, 38923, 38953, 169398,+ 39138, 19251, 39209, 39335, 39362, 39422, 19406, 170800, 39698, 40000, 40189, 19662, 19693,+ 40295, 172238, 19704, 172293, 172558, 172689, 40635, 19798, 40697, 40702, 40709, 40719, 40726,+ 40763, 173568,+};++// compat* hold the full compatibility decomposition (NFKD) of every code+// point where it differs from the canonical one.+inline constexpr uint32_t k_compat_keys[] = {+ 160, 168, 170, 175, 178, 179, 180, 181, 184, 185, 186, 188, 189, 190, 306, 307, 319, 320, 329,+ 383, 452, 453, 454, 455, 456, 457, 458, 459, 460, 497, 498, 499, 688, 689, 690, 691, 692, 693,+ 694, 695, 696, 728, 729, 730, 731, 732, 733, 736, 737, 738, 739, 740, 890, 900, 901, 976, 977,+ 978, 979, 980, 981, 982, 1008, 1009, 1010, 1012, 1013, 1017, 1415, 1653, 1654, 1655, 1656,+ 3635, 3763, 3804, 3805, 3852, 3959, 3961, 4348, 7468, 7469, 7470, 7472, 7473, 7474, 7475, 7476,+ 7477, 7478, 7479, 7480, 7481, 7482, 7484, 7485, 7486, 7487, 7488, 7489, 7490, 7491, 7492, 7493,+ 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510,+ 7511, 7512, 7513, 7514, 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, 7526,+ 7527, 7528, 7529, 7530, 7544, 7579, 7580, 7581, 7582, 7583, 7584, 7585, 7586, 7587, 7588, 7589,+ 7590, 7591, 7592, 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, 7605,+ 7606, 7607, 7608, 7609, 7610, 7611, 7612, 7613, 7614, 7615, 7834, 7835, 8125, 8127, 8128, 8129,+ 8141, 8142, 8143, 8157, 8158, 8159, 8173, 8174, 8189, 8190, 8192, 8193, 8194, 8195, 8196, 8197,+ 8198, 8199, 8200, 8201, 8202, 8209, 8215, 8228, 8229, 8230, 8239, 8243, 8244, 8246, 8247, 8252,+ 8254, 8263, 8264, 8265, 8279, 8287, 8304, 8305, 8308, 8309, 8310, 8311, 8312, 8313, 8314, 8315,+ 8316, 8317, 8318, 8319, 8320, 8321, 8322, 8323, 8324, 8325, 8326, 8327, 8328, 8329, 8330, 8331,+ 8332, 8333, 8334, 8336, 8337, 8338, 8339, 8340, 8341, 8342, 8343, 8344, 8345, 8346, 8347, 8348,+ 8360, 8448, 8449, 8450, 8451, 8453, 8454, 8455, 8457, 8458, 8459, 8460, 8461, 8462, 8463, 8464,+ 8465, 8466, 8467, 8469, 8470, 8473, 8474, 8475, 8476, 8477, 8480, 8481, 8482, 8484, 8488, 8492,+ 8493, 8495, 8496, 8497, 8499, 8500, 8501, 8502, 8503, 8504, 8505, 8507, 8508, 8509, 8510, 8511,+ 8512, 8517, 8518, 8519, 8520, 8521, 8528, 8529, 8530, 8531, 8532, 8533, 8534, 8535, 8536, 8537,+ 8538, 8539, 8540, 8541, 8542, 8543, 8544, 8545, 8546, 8547, 8548, 8549, 8550, 8551, 8552, 8553,+ 8554, 8555, 8556, 8557, 8558, 8559, 8560, 8561, 8562, 8563, 8564, 8565, 8566, 8567, 8568, 8569,+ 8570, 8571, 8572, 8573, 8574, 8575, 8585, 8748, 8749, 8751, 8752, 9312, 9313, 9314, 9315, 9316,+ 9317, 9318, 9319, 9320, 9321, 9322, 9323, 9324, 9325, 9326, 9327, 9328, 9329, 9330, 9331, 9332,+ 9333, 9334, 9335, 9336, 9337, 9338, 9339, 9340, 9341, 9342, 9343, 9344, 9345, 9346, 9347, 9348,+ 9349, 9350, 9351, 9352, 9353, 9354, 9355, 9356, 9357, 9358, 9359, 9360, 9361, 9362, 9363, 9364,+ 9365, 9366, 9367, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380,+ 9381, 9382, 9383, 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, 9393, 9394, 9395, 9396,+ 9397, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410, 9411, 9412,+ 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, 9424, 9425, 9426, 9427, 9428,+ 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444,+ 9445, 9446, 9447, 9448, 9449, 9450, 10764, 10868, 10869, 10870, 11388, 11389, 11631, 11935,+ 12019, 12032, 12033, 12034, 12035, 12036, 12037, 12038, 12039, 12040, 12041, 12042, 12043,+ 12044, 12045, 12046, 12047, 12048, 12049, 12050, 12051, 12052, 12053, 12054, 12055, 12056,+ 12057, 12058, 12059, 12060, 12061, 12062, 12063, 12064, 12065, 12066, 12067, 12068, 12069,+ 12070, 12071, 12072, 12073, 12074, 12075, 12076, 12077, 12078, 12079, 12080, 12081, 12082,+ 12083, 12084, 12085, 12086, 12087, 12088, 12089, 12090, 12091, 12092, 12093, 12094, 12095,+ 12096, 12097, 12098, 12099, 12100, 12101, 12102, 12103, 12104, 12105, 12106, 12107, 12108,+ 12109, 12110, 12111, 12112, 12113, 12114, 12115, 12116, 12117, 12118, 12119, 12120, 12121,+ 12122, 12123, 12124, 12125, 12126, 12127, 12128, 12129, 12130, 12131, 12132, 12133, 12134,+ 12135, 12136, 12137, 12138, 12139, 12140, 12141, 12142, 12143, 12144, 12145, 12146, 12147,+ 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155, 12156, 12157, 12158, 12159, 12160,+ 12161, 12162, 12163, 12164, 12165, 12166, 12167, 12168, 12169, 12170, 12171, 12172, 12173,+ 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, 12184, 12185, 12186,+ 12187, 12188, 12189, 12190, 12191, 12192, 12193, 12194, 12195, 12196, 12197, 12198, 12199,+ 12200, 12201, 12202, 12203, 12204, 12205, 12206, 12207, 12208, 12209, 12210, 12211, 12212,+ 12213, 12214, 12215, 12216, 12217, 12218, 12219, 12220, 12221, 12222, 12223, 12224, 12225,+ 12226, 12227, 12228, 12229, 12230, 12231, 12232, 12233, 12234, 12235, 12236, 12237, 12238,+ 12239, 12240, 12241, 12242, 12243, 12244, 12245, 12288, 12342, 12344, 12345, 12346, 12443,+ 12444, 12447, 12543, 12593, 12594, 12595, 12596, 12597, 12598, 12599, 12600, 12601, 12602,+ 12603, 12604, 12605, 12606, 12607, 12608, 12609, 12610, 12611, 12612, 12613, 12614, 12615,+ 12616, 12617, 12618, 12619, 12620, 12621, 12622, 12623, 12624, 12625, 12626, 12627, 12628,+ 12629, 12630, 12631, 12632, 12633, 12634, 12635, 12636, 12637, 12638, 12639, 12640, 12641,+ 12642, 12643, 12644, 12645, 12646, 12647, 12648, 12649, 12650, 12651, 12652, 12653, 12654,+ 12655, 12656, 12657, 12658, 12659, 12660, 12661, 12662, 12663, 12664, 12665, 12666, 12667,+ 12668, 12669, 12670, 12671, 12672, 12673, 12674, 12675, 12676, 12677, 12678, 12679, 12680,+ 12681, 12682, 12683, 12684, 12685, 12686, 12690, 12691, 12692, 12693, 12694, 12695, 12696,+ 12697, 12698, 12699, 12700, 12701, 12702, 12703, 12800, 12801, 12802, 12803, 12804, 12805,+ 12806, 12807, 12808, 12809, 12810, 12811, 12812, 12813, 12814, 12815, 12816, 12817, 12818,+ 12819, 12820, 12821, 12822, 12823, 12824, 12825, 12826, 12827, 12828, 12829, 12830, 12832,+ 12833, 12834, 12835, 12836, 12837, 12838, 12839, 12840, 12841, 12842, 12843, 12844, 12845,+ 12846, 12847, 12848, 12849, 12850, 12851, 12852, 12853, 12854, 12855, 12856, 12857, 12858,+ 12859, 12860, 12861, 12862, 12863, 12864, 12865, 12866, 12867, 12868, 12869, 12870, 12871,+ 12880, 12881, 12882, 12883, 12884, 12885, 12886, 12887, 12888, 12889, 12890, 12891, 12892,+ 12893, 12894, 12895, 12896, 12897, 12898, 12899, 12900, 12901, 12902, 12903, 12904, 12905,+ 12906, 12907, 12908, 12909, 12910, 12911, 12912, 12913, 12914, 12915, 12916, 12917, 12918,+ 12919, 12920, 12921, 12922, 12923, 12924, 12925, 12926, 12928, 12929, 12930, 12931, 12932,+ 12933, 12934, 12935, 12936, 12937, 12938, 12939, 12940, 12941, 12942, 12943, 12944, 12945,+ 12946, 12947, 12948, 12949, 12950, 12951, 12952, 12953, 12954, 12955, 12956, 12957, 12958,+ 12959, 12960, 12961, 12962, 12963, 12964, 12965, 12966, 12967, 12968, 12969, 12970, 12971,+ 12972, 12973, 12974, 12975, 12976, 12977, 12978, 12979, 12980, 12981, 12982, 12983, 12984,+ 12985, 12986, 12987, 12988, 12989, 12990, 12991, 12992, 12993, 12994, 12995, 12996, 12997,+ 12998, 12999, 13000, 13001, 13002, 13003, 13004, 13005, 13006, 13007, 13008, 13009, 13010,+ 13011, 13012, 13013, 13014, 13015, 13016, 13017, 13018, 13019, 13020, 13021, 13022, 13023,+ 13024, 13025, 13026, 13027, 13028, 13029, 13030, 13031, 13032, 13033, 13034, 13035, 13036,+ 13037, 13038, 13039, 13040, 13041, 13042, 13043, 13044, 13045, 13046, 13047, 13048, 13049,+ 13050, 13051, 13052, 13053, 13054, 13056, 13057, 13058, 13059, 13060, 13061, 13062, 13063,+ 13064, 13065, 13066, 13067, 13068, 13069, 13070, 13071, 13072, 13073, 13074, 13075, 13076,+ 13077, 13078, 13079, 13080, 13081, 13082, 13083, 13084, 13085, 13086, 13087, 13088, 13089,+ 13090, 13091, 13092, 13093, 13094, 13095, 13096, 13097, 13098, 13099, 13100, 13101, 13102,+ 13103, 13104, 13105, 13106, 13107, 13108, 13109, 13110, 13111, 13112, 13113, 13114, 13115,+ 13116, 13117, 13118, 13119, 13120, 13121, 13122, 13123, 13124, 13125, 13126, 13127, 13128,+ 13129, 13130, 13131, 13132, 13133, 13134, 13135, 13136, 13137, 13138, 13139, 13140, 13141,+ 13142, 13143, 13144, 13145, 13146, 13147, 13148, 13149, 13150, 13151, 13152, 13153, 13154,+ 13155, 13156, 13157, 13158, 13159, 13160, 13161, 13162, 13163, 13164, 13165, 13166, 13167,+ 13168, 13169, 13170, 13171, 13172, 13173, 13174, 13175, 13176, 13177, 13178, 13179, 13180,+ 13181, 13182, 13183, 13184, 13185, 13186, 13187, 13188, 13189, 13190, 13191, 13192, 13193,+ 13194, 13195, 13196, 13197, 13198, 13199, 13200, 13201, 13202, 13203, 13204, 13205, 13206,+ 13207, 13208, 13209, 13210, 13211, 13212, 13213, 13214, 13215, 13216, 13217, 13218, 13219,+ 13220, 13221, 13222, 13223, 13224, 13225, 13226, 13227, 13228, 13229, 13230, 13231, 13232,+ 13233, 13234, 13235, 13236, 13237, 13238, 13239, 13240, 13241, 13242, 13243, 13244, 13245,+ 13246, 13247, 13248, 13249, 13250, 13251, 13252, 13253, 13254, 13255, 13256, 13257, 13258,+ 13259, 13260, 13261, 13262, 13263, 13264, 13265, 13266, 13267, 13268, 13269, 13270, 13271,+ 13272, 13273, 13274, 13275, 13276, 13277, 13278, 13279, 13280, 13281, 13282, 13283, 13284,+ 13285, 13286, 13287, 13288, 13289, 13290, 13291, 13292, 13293, 13294, 13295, 13296, 13297,+ 13298, 13299, 13300, 13301, 13302, 13303, 13304, 13305, 13306, 13307, 13308, 13309, 13310,+ 13311, 42652, 42653, 42864, 43000, 43001, 43868, 43869, 43870, 43871, 64256, 64257, 64258,+ 64259, 64260, 64261, 64262, 64275, 64276, 64277, 64278, 64279, 64288, 64289, 64290, 64291,+ 64292, 64293, 64294, 64295, 64296, 64297, 64335, 64336, 64337, 64338, 64339, 64340, 64341,+ 64342, 64343, 64344, 64345, 64346, 64347, 64348, 64349, 64350, 64351, 64352, 64353, 64354,+ 64355, 64356, 64357, 64358, 64359, 64360, 64361, 64362, 64363, 64364, 64365, 64366, 64367,+ 64368, 64369, 64370, 64371, 64372, 64373, 64374, 64375, 64376, 64377, 64378, 64379, 64380,+ 64381, 64382, 64383, 64384, 64385, 64386, 64387, 64388, 64389, 64390, 64391, 64392, 64393,+ 64394, 64395, 64396, 64397, 64398, 64399, 64400, 64401, 64402, 64403, 64404, 64405, 64406,+ 64407, 64408, 64409, 64410, 64411, 64412, 64413, 64414, 64415, 64416, 64417, 64418, 64419,+ 64420, 64421, 64422, 64423, 64424, 64425, 64426, 64427, 64428, 64429, 64430, 64431, 64432,+ 64433, 64467, 64468, 64469, 64470, 64471, 64472, 64473, 64474, 64475, 64476, 64477, 64478,+ 64479, 64480, 64481, 64482, 64483, 64484, 64485, 64486, 64487, 64488, 64489, 64490, 64491,+ 64492, 64493, 64494, 64495, 64496, 64497, 64498, 64499, 64500, 64501, 64502, 64503, 64504,+ 64505, 64506, 64507, 64508, 64509, 64510, 64511, 64512, 64513, 64514, 64515, 64516, 64517,+ 64518, 64519, 64520, 64521, 64522, 64523, 64524, 64525, 64526, 64527, 64528, 64529, 64530,+ 64531, 64532, 64533, 64534, 64535, 64536, 64537, 64538, 64539, 64540, 64541, 64542, 64543,+ 64544, 64545, 64546, 64547, 64548, 64549, 64550, 64551, 64552, 64553, 64554, 64555, 64556,+ 64557, 64558, 64559, 64560, 64561, 64562, 64563, 64564, 64565, 64566, 64567, 64568, 64569,+ 64570, 64571, 64572, 64573, 64574, 64575, 64576, 64577, 64578, 64579, 64580, 64581, 64582,+ 64583, 64584, 64585, 64586, 64587, 64588, 64589, 64590, 64591, 64592, 64593, 64594, 64595,+ 64596, 64597, 64598, 64599, 64600, 64601, 64602, 64603, 64604, 64605, 64606, 64607, 64608,+ 64609, 64610, 64611, 64612, 64613, 64614, 64615, 64616, 64617, 64618, 64619, 64620, 64621,+ 64622, 64623, 64624, 64625, 64626, 64627, 64628, 64629, 64630, 64631, 64632, 64633, 64634,+ 64635, 64636, 64637, 64638, 64639, 64640, 64641, 64642, 64643, 64644, 64645, 64646, 64647,+ 64648, 64649, 64650, 64651, 64652, 64653, 64654, 64655, 64656, 64657, 64658, 64659, 64660,+ 64661, 64662, 64663, 64664, 64665, 64666, 64667, 64668, 64669, 64670, 64671, 64672, 64673,+ 64674, 64675, 64676, 64677, 64678, 64679, 64680, 64681, 64682, 64683, 64684, 64685, 64686,+ 64687, 64688, 64689, 64690, 64691, 64692, 64693, 64694, 64695, 64696, 64697, 64698, 64699,+ 64700, 64701, 64702, 64703, 64704, 64705, 64706, 64707, 64708, 64709, 64710, 64711, 64712,+ 64713, 64714, 64715, 64716, 64717, 64718, 64719, 64720, 64721, 64722, 64723, 64724, 64725,+ 64726, 64727, 64728, 64729, 64730, 64731, 64732, 64733, 64734, 64735, 64736, 64737, 64738,+ 64739, 64740, 64741, 64742, 64743, 64744, 64745, 64746, 64747, 64748, 64749, 64750, 64751,+ 64752, 64753, 64754, 64755, 64756, 64757, 64758, 64759, 64760, 64761, 64762, 64763, 64764,+ 64765, 64766, 64767, 64768, 64769, 64770, 64771, 64772, 64773, 64774, 64775, 64776, 64777,+ 64778, 64779, 64780, 64781, 64782, 64783, 64784, 64785, 64786, 64787, 64788, 64789, 64790,+ 64791, 64792, 64793, 64794, 64795, 64796, 64797, 64798, 64799, 64800, 64801, 64802, 64803,+ 64804, 64805, 64806, 64807, 64808, 64809, 64810, 64811, 64812, 64813, 64814, 64815, 64816,+ 64817, 64818, 64819, 64820, 64821, 64822, 64823, 64824, 64825, 64826, 64827, 64828, 64829,+ 64848, 64849, 64850, 64851, 64852, 64853, 64854, 64855, 64856, 64857, 64858, 64859, 64860,+ 64861, 64862, 64863, 64864, 64865, 64866, 64867, 64868, 64869, 64870, 64871, 64872, 64873,+ 64874, 64875, 64876, 64877, 64878, 64879, 64880, 64881, 64882, 64883, 64884, 64885, 64886,+ 64887, 64888, 64889, 64890, 64891, 64892, 64893, 64894, 64895, 64896, 64897, 64898, 64899,+ 64900, 64901, 64902, 64903, 64904, 64905, 64906, 64907, 64908, 64909, 64910, 64911, 64914,+ 64915, 64916, 64917, 64918, 64919, 64920, 64921, 64922, 64923, 64924, 64925, 64926, 64927,+ 64928, 64929, 64930, 64931, 64932, 64933, 64934, 64935, 64936, 64937, 64938, 64939, 64940,+ 64941, 64942, 64943, 64944, 64945, 64946, 64947, 64948, 64949, 64950, 64951, 64952, 64953,+ 64954, 64955, 64956, 64957, 64958, 64959, 64960, 64961, 64962, 64963, 64964, 64965, 64966,+ 64967, 65008, 65009, 65010, 65011, 65012, 65013, 65014, 65015, 65016, 65017, 65018, 65019,+ 65020, 65040, 65041, 65042, 65043, 65044, 65045, 65046, 65047, 65048, 65049, 65072, 65073,+ 65074, 65075, 65076, 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086,+ 65087, 65088, 65089, 65090, 65091, 65092, 65095, 65096, 65097, 65098, 65099, 65100, 65101,+ 65102, 65103, 65104, 65105, 65106, 65108, 65109, 65110, 65111, 65112, 65113, 65114, 65115,+ 65116, 65117, 65118, 65119, 65120, 65121, 65122, 65123, 65124, 65125, 65126, 65128, 65129,+ 65130, 65131, 65136, 65137, 65138, 65140, 65142, 65143, 65144, 65145, 65146, 65147, 65148,+ 65149, 65150, 65151, 65152, 65153, 65154, 65155, 65156, 65157, 65158, 65159, 65160, 65161,+ 65162, 65163, 65164, 65165, 65166, 65167, 65168, 65169, 65170, 65171, 65172, 65173, 65174,+ 65175, 65176, 65177, 65178, 65179, 65180, 65181, 65182, 65183, 65184, 65185, 65186, 65187,+ 65188, 65189, 65190, 65191, 65192, 65193, 65194, 65195, 65196, 65197, 65198, 65199, 65200,+ 65201, 65202, 65203, 65204, 65205, 65206, 65207, 65208, 65209, 65210, 65211, 65212, 65213,+ 65214, 65215, 65216, 65217, 65218, 65219, 65220, 65221, 65222, 65223, 65224, 65225, 65226,+ 65227, 65228, 65229, 65230, 65231, 65232, 65233, 65234, 65235, 65236, 65237, 65238, 65239,+ 65240, 65241, 65242, 65243, 65244, 65245, 65246, 65247, 65248, 65249, 65250, 65251, 65252,+ 65253, 65254, 65255, 65256, 65257, 65258, 65259, 65260, 65261, 65262, 65263, 65264, 65265,+ 65266, 65267, 65268, 65269, 65270, 65271, 65272, 65273, 65274, 65275, 65276, 65281, 65282,+ 65283, 65284, 65285, 65286, 65287, 65288, 65289, 65290, 65291, 65292, 65293, 65294, 65295,+ 65296, 65297, 65298, 65299, 65300, 65301, 65302, 65303, 65304, 65305, 65306, 65307, 65308,+ 65309, 65310, 65311, 65312, 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321,+ 65322, 65323, 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, 65334,+ 65335, 65336, 65337, 65338, 65339, 65340, 65341, 65342, 65343, 65344, 65345, 65346, 65347,+ 65348, 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, 65360,+ 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, 65369, 65370, 65371, 65372, 65373,+ 65374, 65375, 65376, 65377, 65378, 65379, 65380, 65381, 65382, 65383, 65384, 65385, 65386,+ 65387, 65388, 65389, 65390, 65391, 65392, 65393, 65394, 65395, 65396, 65397, 65398, 65399,+ 65400, 65401, 65402, 65403, 65404, 65405, 65406, 65407, 65408, 65409, 65410, 65411, 65412,+ 65413, 65414, 65415, 65416, 65417, 65418, 65419, 65420, 65421, 65422, 65423, 65424, 65425,+ 65426, 65427, 65428, 65429, 65430, 65431, 65432, 65433, 65434, 65435, 65436, 65437, 65438,+ 65439, 65440, 65441, 65442, 65443, 65444, 65445, 65446, 65447, 65448, 65449, 65450, 65451,+ 65452, 65453, 65454, 65455, 65456, 65457, 65458, 65459, 65460, 65461, 65462, 65463, 65464,+ 65465, 65466, 65467, 65468, 65469, 65470, 65474, 65475, 65476, 65477, 65478, 65479, 65482,+ 65483, 65484, 65485, 65486, 65487, 65490, 65491, 65492, 65493, 65494, 65495, 65498, 65499,+ 65500, 65504, 65505, 65506, 65507, 65508, 65509, 65510, 65512, 65513, 65514, 65515, 65516,+ 65517, 65518, 119808, 119809, 119810, 119811, 119812, 119813, 119814, 119815, 119816, 119817,+ 119818, 119819, 119820, 119821, 119822, 119823, 119824, 119825, 119826, 119827, 119828, 119829,+ 119830, 119831, 119832, 119833, 119834, 119835, 119836, 119837, 119838, 119839, 119840, 119841,+ 119842, 119843, 119844, 119845, 119846, 119847, 119848, 119849, 119850, 119851, 119852, 119853,+ 119854, 119855, 119856, 119857, 119858, 119859, 119860, 119861, 119862, 119863, 119864, 119865,+ 119866, 119867, 119868, 119869, 119870, 119871, 119872, 119873, 119874, 119875, 119876, 119877,+ 119878, 119879, 119880, 119881, 119882, 119883, 119884, 119885, 119886, 119887, 119888, 119889,+ 119890, 119891, 119892, 119894, 119895, 119896, 119897, 119898, 119899, 119900, 119901, 119902,+ 119903, 119904, 119905, 119906, 119907, 119908, 119909, 119910, 119911, 119912, 119913, 119914,+ 119915, 119916, 119917, 119918, 119919, 119920, 119921, 119922, 119923, 119924, 119925, 119926,+ 119927, 119928, 119929, 119930, 119931, 119932, 119933, 119934, 119935, 119936, 119937, 119938,+ 119939, 119940, 119941, 119942, 119943, 119944, 119945, 119946, 119947, 119948, 119949, 119950,+ 119951, 119952, 119953, 119954, 119955, 119956, 119957, 119958, 119959, 119960, 119961, 119962,+ 119963, 119964, 119966, 119967, 119970, 119973, 119974, 119977, 119978, 119979, 119980, 119982,+ 119983, 119984, 119985, 119986, 119987, 119988, 119989, 119990, 119991, 119992, 119993, 119995,+ 119997, 119998, 119999, 120000, 120001, 120002, 120003, 120005, 120006, 120007, 120008, 120009,+ 120010, 120011, 120012, 120013, 120014, 120015, 120016, 120017, 120018, 120019, 120020, 120021,+ 120022, 120023, 120024, 120025, 120026, 120027, 120028, 120029, 120030, 120031, 120032, 120033,+ 120034, 120035, 120036, 120037, 120038, 120039, 120040, 120041, 120042, 120043, 120044, 120045,+ 120046, 120047, 120048, 120049, 120050, 120051, 120052, 120053, 120054, 120055, 120056, 120057,+ 120058, 120059, 120060, 120061, 120062, 120063, 120064, 120065, 120066, 120067, 120068, 120069,+ 120071, 120072, 120073, 120074, 120077, 120078, 120079, 120080, 120081, 120082, 120083, 120084,+ 120086, 120087, 120088, 120089, 120090, 120091, 120092, 120094, 120095, 120096, 120097, 120098,+ 120099, 120100, 120101, 120102, 120103, 120104, 120105, 120106, 120107, 120108, 120109, 120110,+ 120111, 120112, 120113, 120114, 120115, 120116, 120117, 120118, 120119, 120120, 120121, 120123,+ 120124, 120125, 120126, 120128, 120129, 120130, 120131, 120132, 120134, 120138, 120139, 120140,+ 120141, 120142, 120143, 120144, 120146, 120147, 120148, 120149, 120150, 120151, 120152, 120153,+ 120154, 120155, 120156, 120157, 120158, 120159, 120160, 120161, 120162, 120163, 120164, 120165,+ 120166, 120167, 120168, 120169, 120170, 120171, 120172, 120173, 120174, 120175, 120176, 120177,+ 120178, 120179, 120180, 120181, 120182, 120183, 120184, 120185, 120186, 120187, 120188, 120189,+ 120190, 120191, 120192, 120193, 120194, 120195, 120196, 120197, 120198, 120199, 120200, 120201,+ 120202, 120203, 120204, 120205, 120206, 120207, 120208, 120209, 120210, 120211, 120212, 120213,+ 120214, 120215, 120216, 120217, 120218, 120219, 120220, 120221, 120222, 120223, 120224, 120225,+ 120226, 120227, 120228, 120229, 120230, 120231, 120232, 120233, 120234, 120235, 120236, 120237,+ 120238, 120239, 120240, 120241, 120242, 120243, 120244, 120245, 120246, 120247, 120248, 120249,+ 120250, 120251, 120252, 120253, 120254, 120255, 120256, 120257, 120258, 120259, 120260, 120261,+ 120262, 120263, 120264, 120265, 120266, 120267, 120268, 120269, 120270, 120271, 120272, 120273,+ 120274, 120275, 120276, 120277, 120278, 120279, 120280, 120281, 120282, 120283, 120284, 120285,+ 120286, 120287, 120288, 120289, 120290, 120291, 120292, 120293, 120294, 120295, 120296, 120297,+ 120298, 120299, 120300, 120301, 120302, 120303, 120304, 120305, 120306, 120307, 120308, 120309,+ 120310, 120311, 120312, 120313, 120314, 120315, 120316, 120317, 120318, 120319, 120320, 120321,+ 120322, 120323, 120324, 120325, 120326, 120327, 120328, 120329, 120330, 120331, 120332, 120333,+ 120334, 120335, 120336, 120337, 120338, 120339, 120340, 120341, 120342, 120343, 120344, 120345,+ 120346, 120347, 120348, 120349, 120350, 120351, 120352, 120353, 120354, 120355, 120356, 120357,+ 120358, 120359, 120360, 120361, 120362, 120363, 120364, 120365, 120366, 120367, 120368, 120369,+ 120370, 120371, 120372, 120373, 120374, 120375, 120376, 120377, 120378, 120379, 120380, 120381,+ 120382, 120383, 120384, 120385, 120386, 120387, 120388, 120389, 120390, 120391, 120392, 120393,+ 120394, 120395, 120396, 120397, 120398, 120399, 120400, 120401, 120402, 120403, 120404, 120405,+ 120406, 120407, 120408, 120409, 120410, 120411, 120412, 120413, 120414, 120415, 120416, 120417,+ 120418, 120419, 120420, 120421, 120422, 120423, 120424, 120425, 120426, 120427, 120428, 120429,+ 120430, 120431, 120432, 120433, 120434, 120435, 120436, 120437, 120438, 120439, 120440, 120441,+ 120442, 120443, 120444, 120445, 120446, 120447, 120448, 120449, 120450, 120451, 120452, 120453,+ 120454, 120455, 120456, 120457, 120458, 120459, 120460, 120461, 120462, 120463, 120464, 120465,+ 120466, 120467, 120468, 120469, 120470, 120471, 120472, 120473, 120474, 120475, 120476, 120477,+ 120478, 120479, 120480, 120481, 120482, 120483, 120484, 120485, 120488, 120489, 120490, 120491,+ 120492, 120493, 120494, 120495, 120496, 120497, 120498, 120499, 120500, 120501, 120502, 120503,+ 120504, 120505, 120506, 120507, 120508, 120509, 120510, 120511, 120512, 120513, 120514, 120515,+ 120516, 120517, 120518, 120519, 120520, 120521, 120522, 120523, 120524, 120525, 120526, 120527,+ 120528, 120529, 120530, 120531, 120532, 120533, 120534, 120535, 120536, 120537, 120538, 120539,+ 120540, 120541, 120542, 120543, 120544, 120545, 120546, 120547, 120548, 120549, 120550, 120551,+ 120552, 120553, 120554, 120555, 120556, 120557, 120558, 120559, 120560, 120561, 120562, 120563,+ 120564, 120565, 120566, 120567, 120568, 120569, 120570, 120571, 120572, 120573, 120574, 120575,+ 120576, 120577, 120578, 120579, 120580, 120581, 120582, 120583, 120584, 120585, 120586, 120587,+ 120588, 120589, 120590, 120591, 120592, 120593, 120594, 120595, 120596, 120597, 120598, 120599,+ 120600, 120601, 120602, 120603, 120604, 120605, 120606, 120607, 120608, 120609, 120610, 120611,+ 120612, 120613, 120614, 120615, 120616, 120617, 120618, 120619, 120620, 120621, 120622, 120623,+ 120624, 120625, 120626, 120627, 120628, 120629, 120630, 120631, 120632, 120633, 120634, 120635,+ 120636, 120637, 120638, 120639, 120640, 120641, 120642, 120643, 120644, 120645, 120646, 120647,+ 120648, 120649, 120650, 120651, 120652, 120653, 120654, 120655, 120656, 120657, 120658, 120659,+ 120660, 120661, 120662, 120663, 120664, 120665, 120666, 120667, 120668, 120669, 120670, 120671,+ 120672, 120673, 120674, 120675, 120676, 120677, 120678, 120679, 120680, 120681, 120682, 120683,+ 120684, 120685, 120686, 120687, 120688, 120689, 120690, 120691, 120692, 120693, 120694, 120695,+ 120696, 120697, 120698, 120699, 120700, 120701, 120702, 120703, 120704, 120705, 120706, 120707,+ 120708, 120709, 120710, 120711, 120712, 120713, 120714, 120715, 120716, 120717, 120718, 120719,+ 120720, 120721, 120722, 120723, 120724, 120725, 120726, 120727, 120728, 120729, 120730, 120731,+ 120732, 120733, 120734, 120735, 120736, 120737, 120738, 120739, 120740, 120741, 120742, 120743,+ 120744, 120745, 120746, 120747, 120748, 120749, 120750, 120751, 120752, 120753, 120754, 120755,+ 120756, 120757, 120758, 120759, 120760, 120761, 120762, 120763, 120764, 120765, 120766, 120767,+ 120768, 120769, 120770, 120771, 120772, 120773, 120774, 120775, 120776, 120777, 120778, 120779,+ 120782, 120783, 120784, 120785, 120786, 120787, 120788, 120789, 120790, 120791, 120792, 120793,+ 120794, 120795, 120796, 120797, 120798, 120799, 120800, 120801, 120802, 120803, 120804, 120805,+ 120806, 120807, 120808, 120809, 120810, 120811, 120812, 120813, 120814, 120815, 120816, 120817,+ 120818, 120819, 120820, 120821, 120822, 120823, 120824, 120825, 120826, 120827, 120828, 120829,+ 120830, 120831, 126464, 126465, 126466, 126467, 126469, 126470, 126471, 126472, 126473, 126474,+ 126475, 126476, 126477, 126478, 126479, 126480, 126481, 126482, 126483, 126484, 126485, 126486,+ 126487, 126488, 126489, 126490, 126491, 126492, 126493, 126494, 126495, 126497, 126498, 126500,+ 126503, 126505, 126506, 126507, 126508, 126509, 126510, 126511, 126512, 126513, 126514, 126516,+ 126517, 126518, 126519, 126521, 126523, 126530, 126535, 126537, 126539, 126541, 126542, 126543,+ 126545, 126546, 126548, 126551, 126553, 126555, 126557, 126559, 126561, 126562, 126564, 126567,+ 126568, 126569, 126570, 126572, 126573, 126574, 126575, 126576, 126577, 126578, 126580, 126581,+ 126582, 126583, 126585, 126586, 126587, 126588, 126590, 126592, 126593, 126594, 126595, 126596,+ 126597, 126598, 126599, 126600, 126601, 126603, 126604, 126605, 126606, 126607, 126608, 126609,+ 126610, 126611, 126612, 126613, 126614, 126615, 126616, 126617, 126618, 126619, 126625, 126626,+ 126627, 126629, 126630, 126631, 126632, 126633, 126635, 126636, 126637, 126638, 126639, 126640,+ 126641, 126642, 126643, 126644, 126645, 126646, 126647, 126648, 126649, 126650, 126651, 127232,+ 127233, 127234, 127235, 127236, 127237, 127238, 127239, 127240, 127241, 127242, 127248, 127249,+ 127250, 127251, 127252, 127253, 127254, 127255, 127256, 127257, 127258, 127259, 127260, 127261,+ 127262, 127263, 127264, 127265, 127266, 127267, 127268, 127269, 127270, 127271, 127272, 127273,+ 127274, 127275, 127276, 127277, 127278, 127280, 127281, 127282, 127283, 127284, 127285, 127286,+ 127287, 127288, 127289, 127290, 127291, 127292, 127293, 127294, 127295, 127296, 127297, 127298,+ 127299, 127300, 127301, 127302, 127303, 127304, 127305, 127306, 127307, 127308, 127309, 127310,+ 127311, 127338, 127339, 127376, 127488, 127489, 127490, 127504, 127505, 127506, 127507, 127508,+ 127509, 127510, 127511, 127512, 127513, 127514, 127515, 127516, 127517, 127518, 127519, 127520,+ 127521, 127522, 127523, 127524, 127525, 127526, 127527, 127528, 127529, 127530, 127531, 127532,+ 127533, 127534, 127535, 127536, 127537, 127538, 127539, 127540, 127541, 127542, 127543, 127544,+ 127545, 127546, 127547, 127552, 127553, 127554, 127555, 127556, 127557, 127558, 127559, 127560,+ 127568, 127569,+};++// offsets into k_compat_pool+inline constexpr uint32_t k_compat_offsets[] = {+ 0, 1, 3, 4, 6, 7, 8, 10, 11, 13, 14, 15, 18, 21, 24, 26, 28, 30, 32, 34, 35, 38, 41, 44, 46,+ 48, 50, 52, 54, 56, 58, 60, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 73, 75, 77, 79, 81, 83, 84,+ 85, 86, 87, 88, 90, 92, 95, 96, 97, 98, 100, 102, 103, 104, 105, 106, 107, 108, 109, 110, 112,+ 114, 116, 118, 120, 122, 124, 126, 128, 129, 132, 135, 136, 137, 138, 139, 140, 141, 142, 143,+ 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162,+ 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181,+ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200,+ 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,+ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 236, 238, 240, 242,+ 244, 247, 250, 253, 256, 259, 262, 265, 268, 271, 273, 275, 276, 277, 278, 279, 280, 281, 282,+ 283, 284, 285, 286, 287, 289, 290, 292, 295, 296, 298, 301, 303, 306, 308, 310, 312, 314, 316,+ 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338,+ 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357,+ 358, 359, 360, 361, 362, 363, 365, 368, 371, 372, 374, 377, 380, 381, 383, 384, 385, 386, 387,+ 388, 389, 390, 391, 392, 393, 394, 396, 397, 398, 399, 400, 401, 403, 406, 408, 409, 410, 411,+ 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 425, 426, 427, 428, 429, 430, 431, 432,+ 433, 434, 435, 438, 441, 445, 448, 451, 454, 457, 460, 463, 466, 469, 472, 475, 478, 481, 483,+ 484, 486, 489, 491, 492, 494, 497, 501, 503, 504, 506, 509, 510, 511, 512, 513, 514, 516, 519,+ 521, 522, 524, 527, 531, 533, 534, 536, 539, 540, 541, 542, 543, 546, 548, 551, 553, 556, 557,+ 558, 559, 560, 561, 562, 563, 564, 565, 567, 569, 571, 573, 575, 577, 579, 581, 583, 585, 587,+ 590, 593, 596, 599, 602, 605, 608, 611, 614, 618, 622, 626, 630, 634, 638, 642, 646, 650, 654,+ 658, 660, 662, 664, 666, 668, 670, 672, 674, 676, 679, 682, 685, 688, 691, 694, 697, 700, 703,+ 706, 709, 712, 715, 718, 721, 724, 727, 730, 733, 736, 739, 742, 745, 748, 751, 754, 757, 760,+ 763, 766, 769, 772, 775, 778, 781, 784, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797,+ 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816,+ 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835,+ 836, 837, 838, 839, 840, 844, 847, 849, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862,+ 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881,+ 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900,+ 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919,+ 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938,+ 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957,+ 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976,+ 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995,+ 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011,+ 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027,+ 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043,+ 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059,+ 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075,+ 1076, 1078, 1080, 1082, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095,+ 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111,+ 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127,+ 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143,+ 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159,+ 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175,+ 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191,+ 1192, 1195, 1198, 1201, 1204, 1207, 1210, 1213, 1216, 1219, 1222, 1225, 1228, 1231, 1234, 1238,+ 1242, 1246, 1250, 1254, 1258, 1262, 1266, 1270, 1274, 1278, 1282, 1286, 1290, 1294, 1301, 1307,+ 1310, 1313, 1316, 1319, 1322, 1325, 1328, 1331, 1334, 1337, 1340, 1343, 1346, 1349, 1352, 1355,+ 1358, 1361, 1364, 1367, 1370, 1373, 1376, 1379, 1382, 1385, 1388, 1391, 1394, 1397, 1400, 1403,+ 1406, 1409, 1412, 1415, 1416, 1417, 1418, 1419, 1422, 1424, 1426, 1428, 1430, 1432, 1434, 1436,+ 1438, 1440, 1442, 1444, 1446, 1448, 1450, 1452, 1453, 1454, 1455, 1456, 1457, 1458, 1459, 1460,+ 1461, 1462, 1463, 1464, 1465, 1466, 1468, 1470, 1472, 1474, 1476, 1478, 1480, 1482, 1484, 1486,+ 1488, 1490, 1492, 1494, 1499, 1503, 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514,+ 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530,+ 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546,+ 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1556, 1558, 1560, 1562, 1564, 1566, 1568, 1570,+ 1572, 1574, 1576, 1578, 1580, 1582, 1584, 1586, 1588, 1590, 1592, 1594, 1596, 1598, 1600, 1602,+ 1605, 1608, 1611, 1613, 1616, 1618, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630,+ 1631, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1646,+ 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662,+ 1663, 1664, 1665, 1666, 1667, 1668, 1673, 1677, 1682, 1685, 1690, 1693, 1696, 1702, 1706, 1709,+ 1712, 1715, 1719, 1723, 1727, 1731, 1735, 1739, 1743, 1749, 1751, 1757, 1763, 1768, 1772, 1778,+ 1784, 1788, 1791, 1794, 1798, 1802, 1807, 1812, 1815, 1818, 1822, 1825, 1828, 1830, 1832, 1835,+ 1838, 1844, 1848, 1853, 1859, 1863, 1866, 1869, 1875, 1879, 1885, 1888, 1893, 1896, 1900, 1903,+ 1907, 1912, 1916, 1921, 1925, 1927, 1932, 1935, 1938, 1942, 1945, 1948, 1951, 1956, 1960, 1962,+ 1968, 1971, 1976, 1980, 1984, 1987, 1990, 1994, 1996, 2000, 2005, 2007, 2013, 2016, 2018, 2020,+ 2022, 2024, 2026, 2028, 2030, 2032, 2034, 2036, 2039, 2042, 2045, 2048, 2051, 2054, 2057, 2060,+ 2063, 2066, 2069, 2072, 2075, 2078, 2081, 2084, 2086, 2088, 2091, 2093, 2095, 2097, 2100, 2103,+ 2105, 2107, 2109, 2111, 2113, 2117, 2119, 2121, 2123, 2125, 2127, 2129, 2131, 2133, 2136, 2140,+ 2142, 2144, 2146, 2148, 2150, 2152, 2154, 2157, 2160, 2163, 2166, 2168, 2170, 2172, 2174, 2176,+ 2178, 2180, 2182, 2184, 2186, 2189, 2192, 2194, 2197, 2200, 2203, 2205, 2208, 2211, 2215, 2217,+ 2220, 2223, 2226, 2229, 2234, 2240, 2242, 2244, 2246, 2248, 2250, 2252, 2254, 2256, 2258, 2260,+ 2262, 2264, 2266, 2268, 2270, 2272, 2274, 2276, 2280, 2282, 2284, 2286, 2290, 2293, 2295, 2297,+ 2299, 2301, 2303, 2305, 2307, 2309, 2311, 2313, 2316, 2318, 2320, 2323, 2326, 2328, 2332, 2335,+ 2337, 2339, 2341, 2343, 2346, 2349, 2351, 2353, 2355, 2357, 2359, 2361, 2363, 2365, 2367, 2370,+ 2373, 2376, 2379, 2382, 2385, 2388, 2391, 2394, 2397, 2400, 2403, 2406, 2409, 2412, 2415, 2418,+ 2421, 2424, 2427, 2430, 2433, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2443, 2444, 2445, 2447,+ 2449, 2451, 2454, 2457, 2459, 2461, 2463, 2465, 2467, 2469, 2471, 2472, 2473, 2474, 2475, 2476,+ 2477, 2478, 2479, 2480, 2481, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493,+ 2494, 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509,+ 2510, 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525,+ 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541,+ 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557,+ 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, 2569, 2571, 2572, 2573, 2574, 2575,+ 2576, 2577, 2578, 2579, 2580, 2581, 2583, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593,+ 2594, 2595, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, 2612,+ 2615, 2618, 2621, 2624, 2627, 2630, 2633, 2636, 2639, 2642, 2645, 2648, 2651, 2654, 2657, 2660,+ 2663, 2664, 2665, 2666, 2667, 2670, 2673, 2676, 2679, 2682, 2684, 2686, 2688, 2690, 2692, 2694,+ 2696, 2698, 2700, 2702, 2704, 2706, 2708, 2710, 2712, 2714, 2716, 2718, 2720, 2722, 2724, 2726,+ 2728, 2730, 2732, 2734, 2736, 2738, 2740, 2742, 2744, 2746, 2748, 2750, 2752, 2754, 2756, 2758,+ 2760, 2762, 2764, 2766, 2768, 2770, 2772, 2774, 2776, 2778, 2780, 2782, 2784, 2786, 2788, 2790,+ 2792, 2794, 2796, 2798, 2800, 2802, 2804, 2806, 2808, 2810, 2812, 2814, 2816, 2818, 2820, 2822,+ 2824, 2826, 2828, 2830, 2832, 2834, 2836, 2838, 2840, 2842, 2844, 2846, 2848, 2850, 2852, 2854,+ 2856, 2858, 2860, 2863, 2866, 2869, 2872, 2875, 2878, 2881, 2884, 2887, 2890, 2893, 2896, 2898,+ 2900, 2902, 2904, 2906, 2908, 2910, 2912, 2914, 2916, 2918, 2920, 2922, 2924, 2926, 2928, 2930,+ 2932, 2934, 2936, 2938, 2940, 2942, 2944, 2946, 2948, 2950, 2952, 2954, 2956, 2958, 2960, 2962,+ 2964, 2966, 2968, 2970, 2972, 2974, 2976, 2978, 2980, 2982, 2984, 2986, 2989, 2992, 2995, 2998,+ 3001, 3003, 3005, 3007, 3009, 3011, 3013, 3015, 3017, 3019, 3021, 3023, 3025, 3027, 3029, 3031,+ 3033, 3035, 3037, 3039, 3041, 3043, 3045, 3047, 3049, 3051, 3053, 3055, 3057, 3059, 3061, 3063,+ 3065, 3067, 3069, 3071, 3073, 3075, 3077, 3079, 3081, 3083, 3085, 3087, 3089, 3091, 3093, 3095,+ 3097, 3099, 3101, 3103, 3105, 3107, 3109, 3111, 3113, 3115, 3117, 3119, 3121, 3123, 3125, 3127,+ 3129, 3131, 3133, 3135, 3138, 3141, 3143, 3145, 3147, 3149, 3151, 3153, 3155, 3157, 3159, 3161,+ 3163, 3165, 3167, 3169, 3171, 3173, 3175, 3178, 3181, 3184, 3186, 3188, 3190, 3192, 3194, 3196,+ 3198, 3200, 3202, 3204, 3206, 3208, 3210, 3212, 3214, 3216, 3218, 3220, 3222, 3224, 3226, 3228,+ 3230, 3232, 3234, 3236, 3238, 3240, 3242, 3244, 3246, 3248, 3250, 3252, 3254, 3256, 3258, 3260,+ 3262, 3264, 3266, 3268, 3270, 3272, 3274, 3276, 3278, 3280, 3282, 3284, 3286, 3288, 3290, 3292,+ 3294, 3296, 3298, 3300, 3302, 3304, 3306, 3308, 3310, 3312, 3314, 3316, 3318, 3320, 3322, 3324,+ 3326, 3328, 3330, 3333, 3336, 3339, 3342, 3345, 3348, 3351, 3354, 3357, 3360, 3363, 3366, 3369,+ 3372, 3375, 3378, 3381, 3384, 3387, 3390, 3393, 3396, 3399, 3402, 3405, 3408, 3411, 3414, 3417,+ 3420, 3423, 3426, 3429, 3432, 3435, 3438, 3441, 3444, 3447, 3450, 3453, 3456, 3459, 3462, 3465,+ 3468, 3471, 3474, 3477, 3480, 3483, 3486, 3489, 3492, 3495, 3498, 3501, 3504, 3507, 3510, 3513,+ 3516, 3519, 3522, 3525, 3528, 3531, 3534, 3537, 3540, 3543, 3546, 3549, 3552, 3555, 3558, 3561,+ 3564, 3567, 3570, 3573, 3576, 3579, 3582, 3585, 3588, 3591, 3594, 3597, 3600, 3603, 3606, 3609,+ 3612, 3615, 3618, 3621, 3624, 3627, 3630, 3633, 3636, 3639, 3642, 3645, 3648, 3651, 3654, 3657,+ 3660, 3663, 3666, 3669, 3672, 3675, 3678, 3681, 3684, 3687, 3690, 3694, 3698, 3702, 3706, 3710,+ 3714, 3718, 3721, 3739, 3747, 3751, 3752, 3753, 3754, 3755, 3756, 3757, 3758, 3759, 3760, 3763,+ 3765, 3766, 3767, 3768, 3769, 3770, 3771, 3772, 3773, 3774, 3775, 3776, 3777, 3778, 3779, 3780,+ 3781, 3782, 3783, 3784, 3785, 3786, 3787, 3789, 3791, 3793, 3795, 3796, 3797, 3798, 3799, 3800,+ 3801, 3802, 3803, 3804, 3805, 3806, 3807, 3808, 3809, 3810, 3811, 3812, 3813, 3814, 3815, 3816,+ 3817, 3818, 3819, 3820, 3821, 3822, 3823, 3824, 3826, 3828, 3830, 3832, 3834, 3836, 3838, 3840,+ 3842, 3844, 3846, 3848, 3850, 3852, 3853, 3855, 3857, 3859, 3861, 3863, 3865, 3867, 3869, 3871,+ 3873, 3875, 3877, 3878, 3879, 3880, 3881, 3882, 3883, 3884, 3885, 3886, 3887, 3888, 3889, 3890,+ 3891, 3892, 3893, 3894, 3895, 3896, 3897, 3898, 3899, 3900, 3901, 3902, 3903, 3904, 3905, 3906,+ 3907, 3908, 3909, 3910, 3911, 3912, 3913, 3914, 3915, 3916, 3917, 3918, 3919, 3920, 3921, 3922,+ 3923, 3924, 3925, 3926, 3927, 3928, 3929, 3930, 3931, 3932, 3933, 3934, 3935, 3936, 3937, 3938,+ 3939, 3940, 3941, 3942, 3943, 3944, 3945, 3946, 3947, 3948, 3949, 3950, 3951, 3952, 3953, 3954,+ 3955, 3956, 3957, 3958, 3959, 3960, 3961, 3962, 3963, 3964, 3965, 3966, 3967, 3968, 3969, 3970,+ 3971, 3972, 3973, 3974, 3975, 3976, 3977, 3978, 3979, 3980, 3981, 3984, 3987, 3990, 3993, 3996,+ 3999, 4001, 4003, 4004, 4005, 4006, 4007, 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016,+ 4017, 4018, 4019, 4020, 4021, 4022, 4023, 4024, 4025, 4026, 4027, 4028, 4029, 4030, 4031, 4032,+ 4033, 4034, 4035, 4036, 4037, 4038, 4039, 4040, 4041, 4042, 4043, 4044, 4045, 4046, 4047, 4048,+ 4049, 4050, 4051, 4052, 4053, 4054, 4055, 4056, 4057, 4058, 4059, 4060, 4061, 4062, 4063, 4064,+ 4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079, 4080,+ 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095, 4096,+ 4097, 4098, 4099, 4100, 4101, 4102, 4103, 4104, 4105, 4106, 4107, 4108, 4109, 4110, 4111, 4112,+ 4113, 4114, 4115, 4116, 4117, 4118, 4119, 4120, 4121, 4122, 4123, 4124, 4125, 4126, 4127, 4128,+ 4129, 4130, 4131, 4132, 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4143, 4144,+ 4145, 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, 4159, 4160,+ 4161, 4162, 4163, 4164, 4165, 4166, 4167, 4168, 4169, 4170, 4171, 4172, 4173, 4174, 4175, 4176,+ 4177, 4178, 4179, 4180, 4181, 4182, 4183, 4184, 4185, 4186, 4187, 4188, 4189, 4190, 4191, 4192,+ 4193, 4194, 4195, 4196, 4197, 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4205, 4206, 4207, 4208,+ 4209, 4210, 4211, 4212, 4213, 4214, 4215, 4216, 4217, 4219, 4220, 4221, 4222, 4223, 4224, 4225,+ 4226, 4227, 4228, 4229, 4230, 4231, 4232, 4233, 4234, 4235, 4236, 4237, 4238, 4239, 4240, 4241,+ 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249, 4250, 4251, 4252, 4253, 4254, 4255, 4256, 4257,+ 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266, 4267, 4268, 4269, 4270, 4271, 4272, 4273,+ 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282, 4283, 4284, 4285, 4286, 4287, 4288, 4289,+ 4290, 4291, 4292, 4293, 4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303, 4304, 4305,+ 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321,+ 4322, 4323, 4324, 4325, 4326, 4327, 4328, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337,+ 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353,+ 4354, 4355, 4356, 4357, 4358, 4359, 4360, 4361, 4362, 4363, 4364, 4365, 4366, 4367, 4368, 4369,+ 4370, 4371, 4372, 4373, 4374, 4375, 4376, 4377, 4378, 4379, 4380, 4381, 4382, 4383, 4384, 4385,+ 4386, 4387, 4388, 4389, 4390, 4391, 4392, 4393, 4394, 4395, 4396, 4397, 4398, 4399, 4400, 4401,+ 4402, 4403, 4404, 4405, 4406, 4407, 4408, 4409, 4410, 4411, 4412, 4413, 4414, 4415, 4416, 4417,+ 4418, 4419, 4420, 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, 4429, 4430, 4431, 4432, 4433,+ 4434, 4435, 4436, 4437, 4438, 4439, 4440, 4441, 4442, 4443, 4444, 4445, 4446, 4447, 4448, 4449,+ 4450, 4451, 4452, 4453, 4454, 4455, 4456, 4457, 4458, 4459, 4460, 4461, 4462, 4463, 4464, 4465,+ 4466, 4467, 4468, 4469, 4470, 4471, 4472, 4473, 4474, 4475, 4476, 4477, 4478, 4479, 4480, 4481,+ 4482, 4483, 4484, 4485, 4486, 4487, 4488, 4489, 4490, 4491, 4492, 4493, 4494, 4495, 4496, 4497,+ 4498, 4499, 4500, 4501, 4502, 4503, 4504, 4505, 4506, 4507, 4508, 4509, 4510, 4511, 4512, 4513,+ 4514, 4515, 4516, 4517, 4518, 4519, 4520, 4521, 4522, 4523, 4524, 4525, 4526, 4527, 4528, 4529,+ 4530, 4531, 4532, 4533, 4534, 4535, 4536, 4537, 4538, 4539, 4540, 4541, 4542, 4543, 4544, 4545,+ 4546, 4547, 4548, 4549, 4550, 4551, 4552, 4553, 4554, 4555, 4556, 4557, 4558, 4559, 4560, 4561,+ 4562, 4563, 4564, 4565, 4566, 4567, 4568, 4569, 4570, 4571, 4572, 4573, 4574, 4575, 4576, 4577,+ 4578, 4579, 4580, 4581, 4582, 4583, 4584, 4585, 4586, 4587, 4588, 4589, 4590, 4591, 4592, 4593,+ 4594, 4595, 4596, 4597, 4598, 4599, 4600, 4601, 4602, 4603, 4604, 4605, 4606, 4607, 4608, 4609,+ 4610, 4611, 4612, 4613, 4614, 4615, 4616, 4617, 4618, 4619, 4620, 4621, 4622, 4623, 4624, 4625,+ 4626, 4627, 4628, 4629, 4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 4640, 4641,+ 4642, 4643, 4644, 4645, 4646, 4647, 4648, 4649, 4650, 4651, 4652, 4653, 4654, 4655, 4656, 4657,+ 4658, 4659, 4660, 4661, 4662, 4663, 4664, 4665, 4666, 4667, 4668, 4669, 4670, 4671, 4672, 4673,+ 4674, 4675, 4676, 4677, 4678, 4679, 4680, 4681, 4682, 4683, 4684, 4685, 4686, 4687, 4688, 4689,+ 4690, 4691, 4692, 4693, 4694, 4695, 4696, 4697, 4698, 4699, 4700, 4701, 4702, 4703, 4704, 4705,+ 4706, 4707, 4708, 4709, 4710, 4711, 4712, 4713, 4714, 4715, 4716, 4717, 4718, 4719, 4720, 4721,+ 4722, 4723, 4724, 4725, 4726, 4727, 4728, 4729, 4730, 4731, 4732, 4733, 4734, 4735, 4736, 4737,+ 4738, 4739, 4740, 4741, 4742, 4743, 4744, 4745, 4746, 4747, 4748, 4749, 4750, 4751, 4752, 4753,+ 4754, 4755, 4756, 4757, 4758, 4759, 4760, 4761, 4762, 4763, 4764, 4765, 4766, 4767, 4768, 4769,+ 4770, 4771, 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, 4780, 4781, 4782, 4783, 4784, 4785,+ 4786, 4787, 4788, 4789, 4790, 4791, 4792, 4793, 4794, 4795, 4796, 4797, 4798, 4799, 4800, 4801,+ 4802, 4803, 4804, 4805, 4806, 4807, 4808, 4809, 4810, 4811, 4812, 4813, 4814, 4815, 4816, 4817,+ 4818, 4819, 4820, 4821, 4822, 4823, 4824, 4825, 4826, 4827, 4828, 4829, 4830, 4831, 4832, 4833,+ 4834, 4835, 4836, 4837, 4838, 4839, 4840, 4841, 4842, 4843, 4844, 4845, 4846, 4847, 4848, 4849,+ 4850, 4851, 4852, 4853, 4854, 4855, 4856, 4857, 4858, 4859, 4860, 4861, 4862, 4863, 4864, 4865,+ 4866, 4867, 4868, 4869, 4870, 4871, 4872, 4873, 4874, 4875, 4876, 4877, 4878, 4879, 4880, 4881,+ 4882, 4883, 4884, 4885, 4886, 4887, 4888, 4889, 4890, 4891, 4892, 4893, 4894, 4895, 4896, 4897,+ 4898, 4899, 4900, 4901, 4902, 4903, 4904, 4905, 4906, 4907, 4908, 4909, 4910, 4911, 4912, 4913,+ 4914, 4915, 4916, 4917, 4918, 4919, 4920, 4921, 4922, 4923, 4924, 4925, 4926, 4927, 4928, 4929,+ 4930, 4931, 4932, 4933, 4934, 4935, 4936, 4937, 4938, 4939, 4940, 4941, 4942, 4943, 4944, 4945,+ 4946, 4947, 4948, 4949, 4950, 4951, 4952, 4953, 4954, 4955, 4956, 4957, 4958, 4959, 4960, 4961,+ 4962, 4963, 4964, 4965, 4966, 4967, 4968, 4969, 4970, 4971, 4972, 4973, 4974, 4975, 4976, 4977,+ 4978, 4979, 4980, 4981, 4982, 4983, 4984, 4985, 4986, 4987, 4988, 4989, 4990, 4991, 4992, 4993,+ 4994, 4995, 4996, 4997, 4998, 4999, 5000, 5001, 5002, 5003, 5004, 5005, 5006, 5007, 5008, 5009,+ 5010, 5011, 5012, 5013, 5014, 5015, 5016, 5017, 5018, 5019, 5020, 5021, 5022, 5023, 5024, 5025,+ 5026, 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041,+ 5042, 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5054, 5055, 5056, 5057,+ 5058, 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073,+ 5074, 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5089,+ 5090, 5091, 5092, 5093, 5094, 5095, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105,+ 5106, 5107, 5108, 5109, 5110, 5111, 5112, 5113, 5114, 5115, 5116, 5117, 5118, 5119, 5120, 5121,+ 5122, 5123, 5124, 5125, 5126, 5127, 5128, 5129, 5130, 5131, 5132, 5133, 5134, 5135, 5136, 5137,+ 5138, 5139, 5140, 5141, 5142, 5143, 5144, 5145, 5146, 5147, 5148, 5149, 5150, 5151, 5152, 5153,+ 5154, 5155, 5156, 5157, 5158, 5159, 5160, 5161, 5162, 5163, 5164, 5165, 5166, 5167, 5168, 5169,+ 5170, 5171, 5172, 5173, 5174, 5175, 5176, 5177, 5178, 5179, 5180, 5181, 5182, 5183, 5184, 5185,+ 5186, 5187, 5188, 5189, 5190, 5191, 5192, 5193, 5194, 5195, 5196, 5197, 5198, 5199, 5200, 5201,+ 5202, 5203, 5204, 5205, 5206, 5207, 5208, 5209, 5210, 5211, 5212, 5213, 5214, 5215, 5216, 5217,+ 5218, 5219, 5220, 5221, 5222, 5223, 5224, 5225, 5226, 5227, 5228, 5229, 5230, 5231, 5232, 5233,+ 5234, 5235, 5236, 5237, 5238, 5239, 5240, 5241, 5242, 5243, 5244, 5245, 5246, 5247, 5248, 5249,+ 5250, 5251, 5252, 5253, 5254, 5255, 5256, 5257, 5258, 5259, 5260, 5261, 5262, 5263, 5264, 5265,+ 5266, 5267, 5268, 5269, 5270, 5271, 5272, 5273, 5274, 5275, 5276, 5277, 5278, 5279, 5280, 5281,+ 5282, 5283, 5284, 5285, 5286, 5287, 5288, 5289, 5290, 5291, 5292, 5293, 5294, 5295, 5296, 5297,+ 5298, 5299, 5300, 5301, 5302, 5303, 5304, 5305, 5306, 5307, 5308, 5309, 5310, 5311, 5312, 5313,+ 5314, 5315, 5316, 5317, 5318, 5319, 5320, 5321, 5322, 5323, 5324, 5325, 5326, 5327, 5328, 5329,+ 5330, 5331, 5332, 5333, 5334, 5335, 5336, 5337, 5338, 5339, 5340, 5341, 5342, 5343, 5344, 5345,+ 5346, 5347, 5348, 5349, 5350, 5351, 5352, 5353, 5354, 5355, 5356, 5357, 5358, 5359, 5360, 5361,+ 5362, 5363, 5364, 5365, 5366, 5368, 5370, 5372, 5374, 5376, 5378, 5380, 5382, 5384, 5386, 5388,+ 5391, 5394, 5397, 5400, 5403, 5406, 5409, 5412, 5415, 5418, 5421, 5424, 5427, 5430, 5433, 5436,+ 5439, 5442, 5445, 5448, 5451, 5454, 5457, 5460, 5463, 5466, 5469, 5470, 5471, 5473, 5475, 5476,+ 5477, 5478, 5479, 5480, 5481, 5482, 5483, 5484, 5485, 5486, 5487, 5488, 5489, 5490, 5491, 5492,+ 5493, 5494, 5495, 5496, 5497, 5498, 5499, 5500, 5501, 5503, 5505, 5507, 5509, 5512, 5514, 5516,+ 5518, 5520, 5522, 5524, 5525, 5526, 5527, 5528, 5530, 5531, 5532, 5533, 5534, 5535, 5536, 5537,+ 5538, 5539, 5540, 5541, 5542, 5543, 5544, 5545, 5546, 5547, 5548, 5549, 5550, 5551, 5552, 5553,+ 5554, 5555, 5556, 5557, 5558, 5559, 5560, 5561, 5562, 5563, 5564, 5565, 5566, 5567, 5568, 5569,+ 5570, 5573, 5576, 5579, 5582, 5585, 5588, 5591, 5594, 5597, 5598, 5599,+};++// the decompositions of k_compat_keys+inline constexpr uint32_t k_compat_pool[] = {+ 32, 32, 776, 97, 32, 772, 50, 51, 32, 769, 956, 32, 807, 49, 111, 49, 8260, 52, 49, 8260, 50,+ 51, 8260, 52, 73, 74, 105, 106, 76, 183, 108, 183, 700, 110, 115, 68, 90, 780, 68, 122, 780,+ 100, 122, 780, 76, 74, 76, 106, 108, 106, 78, 74, 78, 106, 110, 106, 68, 90, 68, 122, 100, 122,+ 104, 614, 106, 114, 633, 635, 641, 119, 121, 32, 774, 32, 775, 32, 778, 32, 808, 32, 771, 32,+ 779, 611, 108, 115, 120, 661, 32, 837, 32, 769, 32, 776, 769, 946, 952, 933, 933, 769, 933,+ 776, 966, 960, 954, 961, 962, 920, 949, 931, 1381, 1410, 1575, 1652, 1608, 1652, 1735, 1652,+ 1610, 1652, 3661, 3634, 3789, 3762, 3755, 3737, 3755, 3745, 3851, 4018, 3953, 3968, 4019, 3953,+ 3968, 4316, 65, 198, 66, 68, 69, 398, 71, 72, 73, 74, 75, 76, 77, 78, 79, 546, 80, 82, 84, 85,+ 87, 97, 592, 593, 7426, 98, 100, 101, 601, 603, 604, 103, 107, 109, 331, 111, 596, 7446, 7447,+ 112, 116, 117, 7453, 623, 118, 7461, 946, 947, 948, 966, 967, 105, 114, 117, 118, 946, 947,+ 961, 966, 967, 1085, 594, 99, 597, 240, 604, 102, 607, 609, 613, 616, 617, 618, 7547, 669, 621,+ 7557, 671, 625, 624, 626, 627, 628, 629, 632, 642, 643, 427, 649, 650, 7452, 651, 652, 122,+ 656, 657, 658, 952, 97, 702, 115, 775, 32, 787, 32, 787, 32, 834, 32, 776, 834, 32, 787, 768,+ 32, 787, 769, 32, 787, 834, 32, 788, 768, 32, 788, 769, 32, 788, 834, 32, 776, 768, 32, 776,+ 769, 32, 769, 32, 788, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 8208, 32, 819, 46, 46, 46,+ 46, 46, 46, 32, 8242, 8242, 8242, 8242, 8242, 8245, 8245, 8245, 8245, 8245, 33, 33, 32, 773,+ 63, 63, 63, 33, 33, 63, 8242, 8242, 8242, 8242, 32, 48, 105, 52, 53, 54, 55, 56, 57, 43, 8722,+ 61, 40, 41, 110, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 43, 8722, 61, 40, 41, 97, 101, 111,+ 120, 601, 104, 107, 108, 109, 110, 112, 115, 116, 82, 115, 97, 47, 99, 97, 47, 115, 67, 176,+ 67, 99, 47, 111, 99, 47, 117, 400, 176, 70, 103, 72, 72, 72, 104, 295, 73, 73, 76, 108, 78, 78,+ 111, 80, 81, 82, 82, 82, 83, 77, 84, 69, 76, 84, 77, 90, 90, 66, 67, 101, 69, 70, 77, 111,+ 1488, 1489, 1490, 1491, 105, 70, 65, 88, 960, 947, 915, 928, 8721, 68, 100, 101, 105, 106, 49,+ 8260, 55, 49, 8260, 57, 49, 8260, 49, 48, 49, 8260, 51, 50, 8260, 51, 49, 8260, 53, 50, 8260,+ 53, 51, 8260, 53, 52, 8260, 53, 49, 8260, 54, 53, 8260, 54, 49, 8260, 56, 51, 8260, 56, 53,+ 8260, 56, 55, 8260, 56, 49, 8260, 73, 73, 73, 73, 73, 73, 73, 86, 86, 86, 73, 86, 73, 73, 86,+ 73, 73, 73, 73, 88, 88, 88, 73, 88, 73, 73, 76, 67, 68, 77, 105, 105, 105, 105, 105, 105, 105,+ 118, 118, 118, 105, 118, 105, 105, 118, 105, 105, 105, 105, 120, 120, 120, 105, 120, 105, 105,+ 108, 99, 100, 109, 48, 8260, 51, 8747, 8747, 8747, 8747, 8747, 8750, 8750, 8750, 8750, 8750,+ 49, 50, 51, 52, 53, 54, 55, 56, 57, 49, 48, 49, 49, 49, 50, 49, 51, 49, 52, 49, 53, 49, 54, 49,+ 55, 49, 56, 49, 57, 50, 48, 40, 49, 41, 40, 50, 41, 40, 51, 41, 40, 52, 41, 40, 53, 41, 40, 54,+ 41, 40, 55, 41, 40, 56, 41, 40, 57, 41, 40, 49, 48, 41, 40, 49, 49, 41, 40, 49, 50, 41, 40, 49,+ 51, 41, 40, 49, 52, 41, 40, 49, 53, 41, 40, 49, 54, 41, 40, 49, 55, 41, 40, 49, 56, 41, 40, 49,+ 57, 41, 40, 50, 48, 41, 49, 46, 50, 46, 51, 46, 52, 46, 53, 46, 54, 46, 55, 46, 56, 46, 57, 46,+ 49, 48, 46, 49, 49, 46, 49, 50, 46, 49, 51, 46, 49, 52, 46, 49, 53, 46, 49, 54, 46, 49, 55, 46,+ 49, 56, 46, 49, 57, 46, 50, 48, 46, 40, 97, 41, 40, 98, 41, 40, 99, 41, 40, 100, 41, 40, 101,+ 41, 40, 102, 41, 40, 103, 41, 40, 104, 41, 40, 105, 41, 40, 106, 41, 40, 107, 41, 40, 108, 41,+ 40, 109, 41, 40, 110, 41, 40, 111, 41, 40, 112, 41, 40, 113, 41, 40, 114, 41, 40, 115, 41, 40,+ 116, 41, 40, 117, 41, 40, 118, 41, 40, 119, 41, 40, 120, 41, 40, 121, 41, 40, 122, 41, 65, 66,+ 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90,+ 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,+ 116, 117, 118, 119, 120, 121, 122, 48, 8747, 8747, 8747, 8747, 58, 58, 61, 61, 61, 61, 61, 61,+ 106, 86, 11617, 27597, 40863, 19968, 20008, 20022, 20031, 20057, 20101, 20108, 20128, 20154,+ 20799, 20837, 20843, 20866, 20886, 20907, 20960, 20981, 20992, 21147, 21241, 21269, 21274,+ 21304, 21313, 21340, 21353, 21378, 21430, 21448, 21475, 22231, 22303, 22763, 22786, 22794,+ 22805, 22823, 22899, 23376, 23424, 23544, 23567, 23586, 23608, 23662, 23665, 24027, 24037,+ 24049, 24062, 24178, 24186, 24191, 24308, 24318, 24331, 24339, 24400, 24417, 24435, 24515,+ 25096, 25142, 25163, 25903, 25908, 25991, 26007, 26020, 26041, 26080, 26085, 26352, 26376,+ 26408, 27424, 27490, 27513, 27571, 27595, 27604, 27611, 27663, 27668, 27700, 28779, 29226,+ 29238, 29243, 29247, 29255, 29273, 29275, 29356, 29572, 29577, 29916, 29926, 29976, 29983,+ 29992, 30000, 30091, 30098, 30326, 30333, 30382, 30399, 30446, 30683, 30690, 30707, 31034,+ 31160, 31166, 31348, 31435, 31481, 31859, 31992, 32566, 32593, 32650, 32701, 32769, 32780,+ 32786, 32819, 32895, 32905, 33251, 33258, 33267, 33276, 33292, 33307, 33311, 33390, 33394,+ 33400, 34381, 34411, 34880, 34892, 34915, 35198, 35211, 35282, 35328, 35895, 35910, 35925,+ 35960, 35997, 36196, 36208, 36275, 36523, 36554, 36763, 36784, 36789, 37009, 37193, 37318,+ 37324, 37329, 38263, 38272, 38428, 38582, 38585, 38632, 38737, 38750, 38754, 38761, 38859,+ 38893, 38899, 38913, 39080, 39131, 39135, 39318, 39321, 39340, 39592, 39640, 39647, 39717,+ 39727, 39730, 39740, 39770, 40165, 40565, 40575, 40613, 40635, 40643, 40653, 40657, 40697,+ 40701, 40718, 40723, 40736, 40763, 40778, 40786, 40845, 40860, 40864, 32, 12306, 21313, 21316,+ 21317, 32, 12441, 32, 12442, 12424, 12426, 12467, 12488, 4352, 4353, 4522, 4354, 4524, 4525,+ 4355, 4356, 4357, 4528, 4529, 4530, 4531, 4532, 4533, 4378, 4358, 4359, 4360, 4385, 4361, 4362,+ 4363, 4364, 4365, 4366, 4367, 4368, 4369, 4370, 4449, 4450, 4451, 4452, 4453, 4454, 4455, 4456,+ 4457, 4458, 4459, 4460, 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, 4469, 4448, 4372, 4373,+ 4551, 4552, 4556, 4558, 4563, 4567, 4569, 4380, 4573, 4575, 4381, 4382, 4384, 4386, 4387, 4391,+ 4393, 4395, 4396, 4397, 4398, 4399, 4402, 4406, 4416, 4423, 4428, 4593, 4594, 4439, 4440, 4441,+ 4484, 4485, 4488, 4497, 4498, 4500, 4510, 4513, 19968, 20108, 19977, 22235, 19978, 20013,+ 19979, 30002, 20057, 19993, 19969, 22825, 22320, 20154, 40, 4352, 41, 40, 4354, 41, 40, 4355,+ 41, 40, 4357, 41, 40, 4358, 41, 40, 4359, 41, 40, 4361, 41, 40, 4363, 41, 40, 4364, 41, 40,+ 4366, 41, 40, 4367, 41, 40, 4368, 41, 40, 4369, 41, 40, 4370, 41, 40, 4352, 4449, 41, 40, 4354,+ 4449, 41, 40, 4355, 4449, 41, 40, 4357, 4449, 41, 40, 4358, 4449, 41, 40, 4359, 4449, 41, 40,+ 4361, 4449, 41, 40, 4363, 4449, 41, 40, 4364, 4449, 41, 40, 4366, 4449, 41, 40, 4367, 4449, 41,+ 40, 4368, 4449, 41, 40, 4369, 4449, 41, 40, 4370, 4449, 41, 40, 4364, 4462, 41, 40, 4363, 4457,+ 4364, 4453, 4523, 41, 40, 4363, 4457, 4370, 4462, 41, 40, 19968, 41, 40, 20108, 41, 40, 19977,+ 41, 40, 22235, 41, 40, 20116, 41, 40, 20845, 41, 40, 19971, 41, 40, 20843, 41, 40, 20061, 41,+ 40, 21313, 41, 40, 26376, 41, 40, 28779, 41, 40, 27700, 41, 40, 26408, 41, 40, 37329, 41, 40,+ 22303, 41, 40, 26085, 41, 40, 26666, 41, 40, 26377, 41, 40, 31038, 41, 40, 21517, 41, 40,+ 29305, 41, 40, 36001, 41, 40, 31069, 41, 40, 21172, 41, 40, 20195, 41, 40, 21628, 41, 40,+ 23398, 41, 40, 30435, 41, 40, 20225, 41, 40, 36039, 41, 40, 21332, 41, 40, 31085, 41, 40,+ 20241, 41, 40, 33258, 41, 40, 33267, 41, 21839, 24188, 25991, 31631, 80, 84, 69, 50, 49, 50,+ 50, 50, 51, 50, 52, 50, 53, 50, 54, 50, 55, 50, 56, 50, 57, 51, 48, 51, 49, 51, 50, 51, 51, 51,+ 52, 51, 53, 4352, 4354, 4355, 4357, 4358, 4359, 4361, 4363, 4364, 4366, 4367, 4368, 4369, 4370,+ 4352, 4449, 4354, 4449, 4355, 4449, 4357, 4449, 4358, 4449, 4359, 4449, 4361, 4449, 4363, 4449,+ 4364, 4449, 4366, 4449, 4367, 4449, 4368, 4449, 4369, 4449, 4370, 4449, 4366, 4449, 4535, 4352,+ 4457, 4364, 4462, 4363, 4468, 4363, 4462, 19968, 20108, 19977, 22235, 20116, 20845, 19971,+ 20843, 20061, 21313, 26376, 28779, 27700, 26408, 37329, 22303, 26085, 26666, 26377, 31038,+ 21517, 29305, 36001, 31069, 21172, 31192, 30007, 22899, 36969, 20778, 21360, 27880, 38917,+ 20241, 20889, 27491, 19978, 20013, 19979, 24038, 21491, 21307, 23447, 23398, 30435, 20225,+ 36039, 21332, 22812, 51, 54, 51, 55, 51, 56, 51, 57, 52, 48, 52, 49, 52, 50, 52, 51, 52, 52,+ 52, 53, 52, 54, 52, 55, 52, 56, 52, 57, 53, 48, 49, 26376, 50, 26376, 51, 26376, 52, 26376, 53,+ 26376, 54, 26376, 55, 26376, 56, 26376, 57, 26376, 49, 48, 26376, 49, 49, 26376, 49, 50, 26376,+ 72, 103, 101, 114, 103, 101, 86, 76, 84, 68, 12450, 12452, 12454, 12456, 12458, 12459, 12461,+ 12463, 12465, 12467, 12469, 12471, 12473, 12475, 12477, 12479, 12481, 12484, 12486, 12488,+ 12490, 12491, 12492, 12493, 12494, 12495, 12498, 12501, 12504, 12507, 12510, 12511, 12512,+ 12513, 12514, 12516, 12518, 12520, 12521, 12522, 12523, 12524, 12525, 12527, 12528, 12529,+ 12530, 12450, 12495, 12442, 12540, 12488, 12450, 12523, 12501, 12449, 12450, 12531, 12504,+ 12442, 12450, 12450, 12540, 12523, 12452, 12491, 12531, 12463, 12441, 12452, 12531, 12481,+ 12454, 12457, 12531, 12456, 12473, 12463, 12540, 12488, 12441, 12456, 12540, 12459, 12540,+ 12458, 12531, 12473, 12458, 12540, 12512, 12459, 12452, 12522, 12459, 12521, 12483, 12488,+ 12459, 12525, 12522, 12540, 12459, 12441, 12525, 12531, 12459, 12441, 12531, 12510, 12461,+ 12441, 12459, 12441, 12461, 12441, 12491, 12540, 12461, 12517, 12522, 12540, 12461, 12441,+ 12523, 12479, 12441, 12540, 12461, 12525, 12461, 12525, 12463, 12441, 12521, 12512, 12461,+ 12525, 12513, 12540, 12488, 12523, 12461, 12525, 12527, 12483, 12488, 12463, 12441, 12521,+ 12512, 12463, 12441, 12521, 12512, 12488, 12531, 12463, 12523, 12475, 12441, 12452, 12525,+ 12463, 12525, 12540, 12493, 12465, 12540, 12473, 12467, 12523, 12490, 12467, 12540, 12507,+ 12442, 12469, 12452, 12463, 12523, 12469, 12531, 12481, 12540, 12512, 12471, 12522, 12531,+ 12463, 12441, 12475, 12531, 12481, 12475, 12531, 12488, 12479, 12441, 12540, 12473, 12486,+ 12441, 12471, 12488, 12441, 12523, 12488, 12531, 12490, 12494, 12494, 12483, 12488, 12495,+ 12452, 12484, 12495, 12442, 12540, 12475, 12531, 12488, 12495, 12442, 12540, 12484, 12495,+ 12441, 12540, 12524, 12523, 12498, 12442, 12450, 12473, 12488, 12523, 12498, 12442, 12463,+ 12523, 12498, 12442, 12467, 12498, 12441, 12523, 12501, 12449, 12521, 12483, 12488, 12441,+ 12501, 12451, 12540, 12488, 12501, 12441, 12483, 12471, 12455, 12523, 12501, 12521, 12531,+ 12504, 12463, 12479, 12540, 12523, 12504, 12442, 12477, 12504, 12442, 12491, 12498, 12504,+ 12523, 12484, 12504, 12442, 12531, 12473, 12504, 12442, 12540, 12471, 12441, 12504, 12441,+ 12540, 12479, 12507, 12442, 12452, 12531, 12488, 12507, 12441, 12523, 12488, 12507, 12531,+ 12507, 12442, 12531, 12488, 12441, 12507, 12540, 12523, 12507, 12540, 12531, 12510, 12452,+ 12463, 12525, 12510, 12452, 12523, 12510, 12483, 12495, 12510, 12523, 12463, 12510, 12531,+ 12471, 12519, 12531, 12511, 12463, 12525, 12531, 12511, 12522, 12511, 12522, 12495, 12441,+ 12540, 12523, 12513, 12459, 12441, 12513, 12459, 12441, 12488, 12531, 12513, 12540, 12488,+ 12523, 12516, 12540, 12488, 12441, 12516, 12540, 12523, 12518, 12450, 12531, 12522, 12483,+ 12488, 12523, 12522, 12521, 12523, 12498, 12442, 12540, 12523, 12540, 12501, 12441, 12523,+ 12524, 12512, 12524, 12531, 12488, 12465, 12441, 12531, 12527, 12483, 12488, 48, 28857, 49,+ 28857, 50, 28857, 51, 28857, 52, 28857, 53, 28857, 54, 28857, 55, 28857, 56, 28857, 57, 28857,+ 49, 48, 28857, 49, 49, 28857, 49, 50, 28857, 49, 51, 28857, 49, 52, 28857, 49, 53, 28857, 49,+ 54, 28857, 49, 55, 28857, 49, 56, 28857, 49, 57, 28857, 50, 48, 28857, 50, 49, 28857, 50, 50,+ 28857, 50, 51, 28857, 50, 52, 28857, 104, 80, 97, 100, 97, 65, 85, 98, 97, 114, 111, 86, 112,+ 99, 100, 109, 100, 109, 50, 100, 109, 51, 73, 85, 24179, 25104, 26157, 21644, 22823, 27491,+ 26126, 27835, 26666, 24335, 20250, 31038, 112, 65, 110, 65, 956, 65, 109, 65, 107, 65, 75, 66,+ 77, 66, 71, 66, 99, 97, 108, 107, 99, 97, 108, 112, 70, 110, 70, 956, 70, 956, 103, 109, 103,+ 107, 103, 72, 122, 107, 72, 122, 77, 72, 122, 71, 72, 122, 84, 72, 122, 956, 108, 109, 108,+ 100, 108, 107, 108, 102, 109, 110, 109, 956, 109, 109, 109, 99, 109, 107, 109, 109, 109, 50,+ 99, 109, 50, 109, 50, 107, 109, 50, 109, 109, 51, 99, 109, 51, 109, 51, 107, 109, 51, 109,+ 8725, 115, 109, 8725, 115, 50, 80, 97, 107, 80, 97, 77, 80, 97, 71, 80, 97, 114, 97, 100, 114,+ 97, 100, 8725, 115, 114, 97, 100, 8725, 115, 50, 112, 115, 110, 115, 956, 115, 109, 115, 112,+ 86, 110, 86, 956, 86, 109, 86, 107, 86, 77, 86, 112, 87, 110, 87, 956, 87, 109, 87, 107, 87,+ 77, 87, 107, 937, 77, 937, 97, 46, 109, 46, 66, 113, 99, 99, 99, 100, 67, 8725, 107, 103, 67,+ 111, 46, 100, 66, 71, 121, 104, 97, 72, 80, 105, 110, 75, 75, 75, 77, 107, 116, 108, 109, 108,+ 110, 108, 111, 103, 108, 120, 109, 98, 109, 105, 108, 109, 111, 108, 80, 72, 112, 46, 109, 46,+ 80, 80, 77, 80, 82, 115, 114, 83, 118, 87, 98, 86, 8725, 109, 65, 8725, 109, 49, 26085, 50,+ 26085, 51, 26085, 52, 26085, 53, 26085, 54, 26085, 55, 26085, 56, 26085, 57, 26085, 49, 48,+ 26085, 49, 49, 26085, 49, 50, 26085, 49, 51, 26085, 49, 52, 26085, 49, 53, 26085, 49, 54,+ 26085, 49, 55, 26085, 49, 56, 26085, 49, 57, 26085, 50, 48, 26085, 50, 49, 26085, 50, 50,+ 26085, 50, 51, 26085, 50, 52, 26085, 50, 53, 26085, 50, 54, 26085, 50, 55, 26085, 50, 56,+ 26085, 50, 57, 26085, 51, 48, 26085, 51, 49, 26085, 103, 97, 108, 1098, 1100, 42863, 294, 339,+ 42791, 43831, 619, 43858, 102, 102, 102, 105, 102, 108, 102, 102, 105, 102, 102, 108, 115, 116,+ 115, 116, 1396, 1398, 1396, 1381, 1396, 1387, 1406, 1398, 1396, 1389, 1506, 1488, 1491, 1492,+ 1499, 1500, 1501, 1512, 1514, 43, 1488, 1500, 1649, 1649, 1659, 1659, 1659, 1659, 1662, 1662,+ 1662, 1662, 1664, 1664, 1664, 1664, 1658, 1658, 1658, 1658, 1663, 1663, 1663, 1663, 1657, 1657,+ 1657, 1657, 1700, 1700, 1700, 1700, 1702, 1702, 1702, 1702, 1668, 1668, 1668, 1668, 1667, 1667,+ 1667, 1667, 1670, 1670, 1670, 1670, 1671, 1671, 1671, 1671, 1677, 1677, 1676, 1676, 1678, 1678,+ 1672, 1672, 1688, 1688, 1681, 1681, 1705, 1705, 1705, 1705, 1711, 1711, 1711, 1711, 1715, 1715,+ 1715, 1715, 1713, 1713, 1713, 1713, 1722, 1722, 1723, 1723, 1723, 1723, 1749, 1620, 1749, 1620,+ 1729, 1729, 1729, 1729, 1726, 1726, 1726, 1726, 1746, 1746, 1746, 1620, 1746, 1620, 1709, 1709,+ 1709, 1709, 1735, 1735, 1734, 1734, 1736, 1736, 1735, 1652, 1739, 1739, 1733, 1733, 1737, 1737,+ 1744, 1744, 1744, 1744, 1609, 1609, 1610, 1620, 1575, 1610, 1620, 1575, 1610, 1620, 1749, 1610,+ 1620, 1749, 1610, 1620, 1608, 1610, 1620, 1608, 1610, 1620, 1735, 1610, 1620, 1735, 1610, 1620,+ 1734, 1610, 1620, 1734, 1610, 1620, 1736, 1610, 1620, 1736, 1610, 1620, 1744, 1610, 1620, 1744,+ 1610, 1620, 1744, 1610, 1620, 1609, 1610, 1620, 1609, 1610, 1620, 1609, 1740, 1740, 1740, 1740,+ 1610, 1620, 1580, 1610, 1620, 1581, 1610, 1620, 1605, 1610, 1620, 1609, 1610, 1620, 1610, 1576,+ 1580, 1576, 1581, 1576, 1582, 1576, 1605, 1576, 1609, 1576, 1610, 1578, 1580, 1578, 1581, 1578,+ 1582, 1578, 1605, 1578, 1609, 1578, 1610, 1579, 1580, 1579, 1605, 1579, 1609, 1579, 1610, 1580,+ 1581, 1580, 1605, 1581, 1580, 1581, 1605, 1582, 1580, 1582, 1581, 1582, 1605, 1587, 1580, 1587,+ 1581, 1587, 1582, 1587, 1605, 1589, 1581, 1589, 1605, 1590, 1580, 1590, 1581, 1590, 1582, 1590,+ 1605, 1591, 1581, 1591, 1605, 1592, 1605, 1593, 1580, 1593, 1605, 1594, 1580, 1594, 1605, 1601,+ 1580, 1601, 1581, 1601, 1582, 1601, 1605, 1601, 1609, 1601, 1610, 1602, 1581, 1602, 1605, 1602,+ 1609, 1602, 1610, 1603, 1575, 1603, 1580, 1603, 1581, 1603, 1582, 1603, 1604, 1603, 1605, 1603,+ 1609, 1603, 1610, 1604, 1580, 1604, 1581, 1604, 1582, 1604, 1605, 1604, 1609, 1604, 1610, 1605,+ 1580, 1605, 1581, 1605, 1582, 1605, 1605, 1605, 1609, 1605, 1610, 1606, 1580, 1606, 1581, 1606,+ 1582, 1606, 1605, 1606, 1609, 1606, 1610, 1607, 1580, 1607, 1605, 1607, 1609, 1607, 1610, 1610,+ 1580, 1610, 1581, 1610, 1582, 1610, 1605, 1610, 1609, 1610, 1610, 1584, 1648, 1585, 1648, 1609,+ 1648, 32, 1612, 1617, 32, 1613, 1617, 32, 1614, 1617, 32, 1615, 1617, 32, 1616, 1617, 32, 1617,+ 1648, 1610, 1620, 1585, 1610, 1620, 1586, 1610, 1620, 1605, 1610, 1620, 1606, 1610, 1620, 1609,+ 1610, 1620, 1610, 1576, 1585, 1576, 1586, 1576, 1605, 1576, 1606, 1576, 1609, 1576, 1610, 1578,+ 1585, 1578, 1586, 1578, 1605, 1578, 1606, 1578, 1609, 1578, 1610, 1579, 1585, 1579, 1586, 1579,+ 1605, 1579, 1606, 1579, 1609, 1579, 1610, 1601, 1609, 1601, 1610, 1602, 1609, 1602, 1610, 1603,+ 1575, 1603, 1604, 1603, 1605, 1603, 1609, 1603, 1610, 1604, 1605, 1604, 1609, 1604, 1610, 1605,+ 1575, 1605, 1605, 1606, 1585, 1606, 1586, 1606, 1605, 1606, 1606, 1606, 1609, 1606, 1610, 1609,+ 1648, 1610, 1585, 1610, 1586, 1610, 1605, 1610, 1606, 1610, 1609, 1610, 1610, 1610, 1620, 1580,+ 1610, 1620, 1581, 1610, 1620, 1582, 1610, 1620, 1605, 1610, 1620, 1607, 1576, 1580, 1576, 1581,+ 1576, 1582, 1576, 1605, 1576, 1607, 1578, 1580, 1578, 1581, 1578, 1582, 1578, 1605, 1578, 1607,+ 1579, 1605, 1580, 1581, 1580, 1605, 1581, 1580, 1581, 1605, 1582, 1580, 1582, 1605, 1587, 1580,+ 1587, 1581, 1587, 1582, 1587, 1605, 1589, 1581, 1589, 1582, 1589, 1605, 1590, 1580, 1590, 1581,+ 1590, 1582, 1590, 1605, 1591, 1581, 1592, 1605, 1593, 1580, 1593, 1605, 1594, 1580, 1594, 1605,+ 1601, 1580, 1601, 1581, 1601, 1582, 1601, 1605, 1602, 1581, 1602, 1605, 1603, 1580, 1603, 1581,+ 1603, 1582, 1603, 1604, 1603, 1605, 1604, 1580, 1604, 1581, 1604, 1582, 1604, 1605, 1604, 1607,+ 1605, 1580, 1605, 1581, 1605, 1582, 1605, 1605, 1606, 1580, 1606, 1581, 1606, 1582, 1606, 1605,+ 1606, 1607, 1607, 1580, 1607, 1605, 1607, 1648, 1610, 1580, 1610, 1581, 1610, 1582, 1610, 1605,+ 1610, 1607, 1610, 1620, 1605, 1610, 1620, 1607, 1576, 1605, 1576, 1607, 1578, 1605, 1578, 1607,+ 1579, 1605, 1579, 1607, 1587, 1605, 1587, 1607, 1588, 1605, 1588, 1607, 1603, 1604, 1603, 1605,+ 1604, 1605, 1606, 1605, 1606, 1607, 1610, 1605, 1610, 1607, 1600, 1614, 1617, 1600, 1615, 1617,+ 1600, 1616, 1617, 1591, 1609, 1591, 1610, 1593, 1609, 1593, 1610, 1594, 1609, 1594, 1610, 1587,+ 1609, 1587, 1610, 1588, 1609, 1588, 1610, 1581, 1609, 1581, 1610, 1580, 1609, 1580, 1610, 1582,+ 1609, 1582, 1610, 1589, 1609, 1589, 1610, 1590, 1609, 1590, 1610, 1588, 1580, 1588, 1581, 1588,+ 1582, 1588, 1605, 1588, 1585, 1587, 1585, 1589, 1585, 1590, 1585, 1591, 1609, 1591, 1610, 1593,+ 1609, 1593, 1610, 1594, 1609, 1594, 1610, 1587, 1609, 1587, 1610, 1588, 1609, 1588, 1610, 1581,+ 1609, 1581, 1610, 1580, 1609, 1580, 1610, 1582, 1609, 1582, 1610, 1589, 1609, 1589, 1610, 1590,+ 1609, 1590, 1610, 1588, 1580, 1588, 1581, 1588, 1582, 1588, 1605, 1588, 1585, 1587, 1585, 1589,+ 1585, 1590, 1585, 1588, 1580, 1588, 1581, 1588, 1582, 1588, 1605, 1587, 1607, 1588, 1607, 1591,+ 1605, 1587, 1580, 1587, 1581, 1587, 1582, 1588, 1580, 1588, 1581, 1588, 1582, 1591, 1605, 1592,+ 1605, 1575, 1611, 1575, 1611, 1578, 1580, 1605, 1578, 1581, 1580, 1578, 1581, 1580, 1578, 1581,+ 1605, 1578, 1582, 1605, 1578, 1605, 1580, 1578, 1605, 1581, 1578, 1605, 1582, 1580, 1605, 1581,+ 1580, 1605, 1581, 1581, 1605, 1610, 1581, 1605, 1609, 1587, 1581, 1580, 1587, 1580, 1581, 1587,+ 1580, 1609, 1587, 1605, 1581, 1587, 1605, 1581, 1587, 1605, 1580, 1587, 1605, 1605, 1587, 1605,+ 1605, 1589, 1581, 1581, 1589, 1581, 1581, 1589, 1605, 1605, 1588, 1581, 1605, 1588, 1581, 1605,+ 1588, 1580, 1610, 1588, 1605, 1582, 1588, 1605, 1582, 1588, 1605, 1605, 1588, 1605, 1605, 1590,+ 1581, 1609, 1590, 1582, 1605, 1590, 1582, 1605, 1591, 1605, 1581, 1591, 1605, 1581, 1591, 1605,+ 1605, 1591, 1605, 1610, 1593, 1580, 1605, 1593, 1605, 1605, 1593, 1605, 1605, 1593, 1605, 1609,+ 1594, 1605, 1605, 1594, 1605, 1610, 1594, 1605, 1609, 1601, 1582, 1605, 1601, 1582, 1605, 1602,+ 1605, 1581, 1602, 1605, 1605, 1604, 1581, 1605, 1604, 1581, 1610, 1604, 1581, 1609, 1604, 1580,+ 1580, 1604, 1580, 1580, 1604, 1582, 1605, 1604, 1582, 1605, 1604, 1605, 1581, 1604, 1605, 1581,+ 1605, 1581, 1580, 1605, 1581, 1605, 1605, 1581, 1610, 1605, 1580, 1581, 1605, 1580, 1605, 1605,+ 1582, 1580, 1605, 1582, 1605, 1605, 1580, 1582, 1607, 1605, 1580, 1607, 1605, 1605, 1606, 1581,+ 1605, 1606, 1581, 1609, 1606, 1580, 1605, 1606, 1580, 1605, 1606, 1580, 1609, 1606, 1605, 1610,+ 1606, 1605, 1609, 1610, 1605, 1605, 1610, 1605, 1605, 1576, 1582, 1610, 1578, 1580, 1610, 1578,+ 1580, 1609, 1578, 1582, 1610, 1578, 1582, 1609, 1578, 1605, 1610, 1578, 1605, 1609, 1580, 1605,+ 1610, 1580, 1581, 1609, 1580, 1605, 1609, 1587, 1582, 1609, 1589, 1581, 1610, 1588, 1581, 1610,+ 1590, 1581, 1610, 1604, 1580, 1610, 1604, 1605, 1610, 1610, 1581, 1610, 1610, 1580, 1610, 1610,+ 1605, 1610, 1605, 1605, 1610, 1602, 1605, 1610, 1606, 1581, 1610, 1602, 1605, 1581, 1604, 1581,+ 1605, 1593, 1605, 1610, 1603, 1605, 1610, 1606, 1580, 1581, 1605, 1582, 1610, 1604, 1580, 1605,+ 1603, 1605, 1605, 1604, 1580, 1605, 1606, 1580, 1581, 1580, 1581, 1610, 1581, 1580, 1610, 1605,+ 1580, 1610, 1601, 1605, 1610, 1576, 1581, 1610, 1603, 1605, 1605, 1593, 1580, 1605, 1589, 1605,+ 1605, 1587, 1582, 1610, 1606, 1580, 1610, 1589, 1604, 1746, 1602, 1604, 1746, 1575, 1604, 1604,+ 1607, 1575, 1603, 1576, 1585, 1605, 1581, 1605, 1583, 1589, 1604, 1593, 1605, 1585, 1587, 1608,+ 1604, 1593, 1604, 1610, 1607, 1608, 1587, 1604, 1605, 1589, 1604, 1609, 1589, 1604, 1609, 32,+ 1575, 1604, 1604, 1607, 32, 1593, 1604, 1610, 1607, 32, 1608, 1587, 1604, 1605, 1580, 1604, 32,+ 1580, 1604, 1575, 1604, 1607, 1585, 1740, 1575, 1604, 44, 12289, 12290, 58, 59, 33, 63, 12310,+ 12311, 46, 46, 46, 46, 46, 8212, 8211, 95, 95, 40, 41, 123, 125, 12308, 12309, 12304, 12305,+ 12298, 12299, 12296, 12297, 12300, 12301, 12302, 12303, 91, 93, 32, 773, 32, 773, 32, 773, 32,+ 773, 95, 95, 95, 44, 12289, 46, 59, 58, 63, 33, 8212, 40, 41, 123, 125, 12308, 12309, 35, 38,+ 42, 43, 45, 60, 62, 61, 92, 36, 37, 64, 32, 1611, 1600, 1611, 32, 1612, 32, 1613, 32, 1614,+ 1600, 1614, 32, 1615, 1600, 1615, 32, 1616, 1600, 1616, 32, 1617, 1600, 1617, 32, 1618, 1600,+ 1618, 1569, 1575, 1619, 1575, 1619, 1575, 1620, 1575, 1620, 1608, 1620, 1608, 1620, 1575, 1621,+ 1575, 1621, 1610, 1620, 1610, 1620, 1610, 1620, 1610, 1620, 1575, 1575, 1576, 1576, 1576, 1576,+ 1577, 1577, 1578, 1578, 1578, 1578, 1579, 1579, 1579, 1579, 1580, 1580, 1580, 1580, 1581, 1581,+ 1581, 1581, 1582, 1582, 1582, 1582, 1583, 1583, 1584, 1584, 1585, 1585, 1586, 1586, 1587, 1587,+ 1587, 1587, 1588, 1588, 1588, 1588, 1589, 1589, 1589, 1589, 1590, 1590, 1590, 1590, 1591, 1591,+ 1591, 1591, 1592, 1592, 1592, 1592, 1593, 1593, 1593, 1593, 1594, 1594, 1594, 1594, 1601, 1601,+ 1601, 1601, 1602, 1602, 1602, 1602, 1603, 1603, 1603, 1603, 1604, 1604, 1604, 1604, 1605, 1605,+ 1605, 1605, 1606, 1606, 1606, 1606, 1607, 1607, 1607, 1607, 1608, 1608, 1609, 1609, 1610, 1610,+ 1610, 1610, 1604, 1575, 1619, 1604, 1575, 1619, 1604, 1575, 1620, 1604, 1575, 1620, 1604, 1575,+ 1621, 1604, 1575, 1621, 1604, 1575, 1604, 1575, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,+ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68,+ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92,+ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112,+ 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 10629, 10630, 12290,+ 12300, 12301, 12289, 12539, 12530, 12449, 12451, 12453, 12455, 12457, 12515, 12517, 12519,+ 12483, 12540, 12450, 12452, 12454, 12456, 12458, 12459, 12461, 12463, 12465, 12467, 12469,+ 12471, 12473, 12475, 12477, 12479, 12481, 12484, 12486, 12488, 12490, 12491, 12492, 12493,+ 12494, 12495, 12498, 12501, 12504, 12507, 12510, 12511, 12512, 12513, 12514, 12516, 12518,+ 12520, 12521, 12522, 12523, 12524, 12525, 12527, 12531, 12441, 12442, 4448, 4352, 4353, 4522,+ 4354, 4524, 4525, 4355, 4356, 4357, 4528, 4529, 4530, 4531, 4532, 4533, 4378, 4358, 4359, 4360,+ 4385, 4361, 4362, 4363, 4364, 4365, 4366, 4367, 4368, 4369, 4370, 4449, 4450, 4451, 4452, 4453,+ 4454, 4455, 4456, 4457, 4458, 4459, 4460, 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, 4469,+ 162, 163, 172, 32, 772, 166, 165, 8361, 9474, 8592, 8593, 8594, 8595, 9632, 9675, 65, 66, 67,+ 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97,+ 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,+ 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80,+ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108,+ 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70,+ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99,+ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,+ 119, 120, 121, 122, 65, 67, 68, 71, 74, 75, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 97,+ 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 112, 113, 114, 115, 116, 117, 118, 119,+ 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84,+ 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111,+ 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 68, 69, 70, 71, 74, 75, 76, 77,+ 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107,+ 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 68, 69, 70,+ 71, 73, 74, 75, 76, 77, 79, 83, 84, 85, 86, 87, 88, 89, 97, 98, 99, 100, 101, 102, 103, 104,+ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65,+ 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,+ 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,+ 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,+ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107,+ 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69,+ 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99,+ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118,+ 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83,+ 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110,+ 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73,+ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102,+ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121,+ 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86,+ 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112,+ 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 305, 567, 913, 914, 915, 916, 917, 918, 919,+ 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934, 935, 936, 937, 8711,+ 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963,+ 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 913, 914, 915, 916, 917, 918,+ 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934, 935, 936, 937,+ 8711, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962,+ 963, 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 913, 914, 915, 916, 917,+ 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934, 935, 936,+ 937, 8711, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961,+ 962, 963, 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 913, 914, 915, 916,+ 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934, 935,+ 936, 937, 8711, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960,+ 961, 962, 963, 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 913, 914, 915,+ 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 920, 931, 932, 933, 934,+ 935, 936, 937, 8711, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959,+ 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 8706, 949, 952, 954, 966, 961, 960, 988, 989,+ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 49, 50, 51,+ 52, 53, 54, 55, 56, 57, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 49, 50, 51, 52, 53, 54, 55,+ 56, 57, 1575, 1576, 1580, 1583, 1608, 1586, 1581, 1591, 1610, 1603, 1604, 1605, 1606, 1587,+ 1593, 1601, 1589, 1602, 1585, 1588, 1578, 1579, 1582, 1584, 1590, 1592, 1594, 1646, 1722, 1697,+ 1647, 1576, 1580, 1607, 1581, 1610, 1603, 1604, 1605, 1606, 1587, 1593, 1601, 1589, 1602, 1588,+ 1578, 1579, 1582, 1590, 1594, 1580, 1581, 1610, 1604, 1606, 1587, 1593, 1589, 1602, 1588, 1582,+ 1590, 1594, 1722, 1647, 1576, 1580, 1607, 1581, 1591, 1610, 1603, 1605, 1606, 1587, 1593, 1601,+ 1589, 1602, 1588, 1578, 1579, 1582, 1590, 1592, 1594, 1646, 1697, 1575, 1576, 1580, 1583, 1607,+ 1608, 1586, 1581, 1591, 1610, 1604, 1605, 1606, 1587, 1593, 1601, 1589, 1602, 1585, 1588, 1578,+ 1579, 1582, 1584, 1590, 1592, 1594, 1576, 1580, 1583, 1608, 1586, 1581, 1591, 1610, 1604, 1605,+ 1606, 1587, 1593, 1601, 1589, 1602, 1585, 1588, 1578, 1579, 1582, 1584, 1590, 1592, 1594, 48,+ 46, 48, 44, 49, 44, 50, 44, 51, 44, 52, 44, 53, 44, 54, 44, 55, 44, 56, 44, 57, 44, 40, 65, 41,+ 40, 66, 41, 40, 67, 41, 40, 68, 41, 40, 69, 41, 40, 70, 41, 40, 71, 41, 40, 72, 41, 40, 73, 41,+ 40, 74, 41, 40, 75, 41, 40, 76, 41, 40, 77, 41, 40, 78, 41, 40, 79, 41, 40, 80, 41, 40, 81, 41,+ 40, 82, 41, 40, 83, 41, 40, 84, 41, 40, 85, 41, 40, 86, 41, 40, 87, 41, 40, 88, 41, 40, 89, 41,+ 40, 90, 41, 12308, 83, 12309, 67, 82, 67, 68, 87, 90, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74,+ 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 72, 86, 77, 86, 83, 68, 83, 83,+ 80, 80, 86, 87, 67, 77, 67, 77, 68, 68, 74, 12411, 12363, 12467, 12467, 12469, 25163, 23383,+ 21452, 12486, 12441, 20108, 22810, 35299, 22825, 20132, 26144, 28961, 26009, 21069, 24460,+ 20877, 26032, 21021, 32066, 29983, 36009, 22768, 21561, 28436, 25237, 25429, 19968, 19977,+ 36938, 24038, 20013, 21491, 25351, 36208, 25171, 31105, 31354, 21512, 28288, 26377, 26376,+ 30003, 21106, 21942, 37197, 12308, 26412, 12309, 12308, 19977, 12309, 12308, 20108, 12309,+ 12308, 23433, 12309, 12308, 28857, 12309, 12308, 25171, 12309, 12308, 30423, 12309, 12308,+ 21213, 12309, 12308, 25943, 12309, 24471, 21487,+};++// cccRanges are (lo, hi, class) triples of the canonical combining class, sorted.+inline constexpr uint32_t k_ccc_ranges[] = {+ 768, 788, 230, 789, 789, 232, 790, 793, 220, 794, 794, 232, 795, 795, 216, 796, 800, 220, 801,+ 802, 202, 803, 806, 220, 807, 808, 202, 809, 819, 220, 820, 824, 1, 825, 828, 220, 829, 831,+ 230, 834, 834, 230, 837, 837, 240, 838, 838, 230, 839, 841, 220, 842, 844, 230, 845, 846, 220,+ 848, 850, 230, 851, 854, 220, 855, 855, 230, 856, 856, 232, 857, 858, 220, 859, 859, 230, 860,+ 860, 233, 861, 862, 234, 863, 863, 233, 864, 865, 234, 866, 866, 233, 867, 879, 230, 1155,+ 1159, 230, 1425, 1425, 220, 1426, 1429, 230, 1430, 1430, 220, 1431, 1433, 230, 1434, 1434, 222,+ 1435, 1435, 220, 1436, 1441, 230, 1442, 1447, 220, 1448, 1449, 230, 1450, 1450, 220, 1451,+ 1452, 230, 1453, 1453, 222, 1454, 1454, 228, 1455, 1455, 230, 1456, 1456, 10, 1457, 1457, 11,+ 1458, 1458, 12, 1459, 1459, 13, 1460, 1460, 14, 1461, 1461, 15, 1462, 1462, 16, 1463, 1463, 17,+ 1464, 1464, 18, 1465, 1466, 19, 1467, 1467, 20, 1468, 1468, 21, 1469, 1469, 22, 1471, 1471, 23,+ 1473, 1473, 24, 1474, 1474, 25, 1476, 1476, 230, 1477, 1477, 220, 1479, 1479, 18, 1552, 1559,+ 230, 1560, 1560, 30, 1561, 1561, 31, 1562, 1562, 32, 1611, 1611, 27, 1612, 1612, 28, 1613,+ 1613, 29, 1614, 1614, 30, 1615, 1615, 31, 1616, 1616, 32, 1617, 1617, 33, 1618, 1618, 34, 1619,+ 1620, 230, 1621, 1622, 220, 1623, 1627, 230, 1628, 1628, 220, 1629, 1630, 230, 1631, 1631, 220,+ 1648, 1648, 35, 1750, 1756, 230, 1759, 1762, 230, 1763, 1763, 220, 1764, 1764, 230, 1767, 1768,+ 230, 1770, 1770, 220, 1771, 1772, 230, 1773, 1773, 220, 1809, 1809, 36, 1840, 1840, 230, 1841,+ 1841, 220, 1842, 1843, 230, 1844, 1844, 220, 1845, 1846, 230, 1847, 1849, 220, 1850, 1850, 230,+ 1851, 1852, 220, 1853, 1853, 230, 1854, 1854, 220, 1855, 1857, 230, 1858, 1858, 220, 1859,+ 1859, 230, 1860, 1860, 220, 1861, 1861, 230, 1862, 1862, 220, 1863, 1863, 230, 1864, 1864, 220,+ 1865, 1866, 230, 2027, 2033, 230, 2034, 2034, 220, 2035, 2035, 230, 2070, 2073, 230, 2075,+ 2083, 230, 2085, 2087, 230, 2089, 2093, 230, 2137, 2139, 220, 2260, 2273, 230, 2275, 2275, 220,+ 2276, 2277, 230, 2278, 2278, 220, 2279, 2280, 230, 2281, 2281, 220, 2282, 2284, 230, 2285,+ 2287, 220, 2288, 2288, 27, 2289, 2289, 28, 2290, 2290, 29, 2291, 2293, 230, 2294, 2294, 220,+ 2295, 2296, 230, 2297, 2298, 220, 2299, 2303, 230, 2364, 2364, 7, 2381, 2381, 9, 2385, 2385,+ 230, 2386, 2386, 220, 2387, 2388, 230, 2492, 2492, 7, 2509, 2509, 9, 2620, 2620, 7, 2637, 2637,+ 9, 2748, 2748, 7, 2765, 2765, 9, 2876, 2876, 7, 2893, 2893, 9, 3021, 3021, 9, 3149, 3149, 9,+ 3157, 3157, 84, 3158, 3158, 91, 3260, 3260, 7, 3277, 3277, 9, 3405, 3405, 9, 3530, 3530, 9,+ 3640, 3641, 103, 3642, 3642, 9, 3656, 3659, 107, 3768, 3769, 118, 3784, 3787, 122, 3864, 3865,+ 220, 3893, 3893, 220, 3895, 3895, 220, 3897, 3897, 216, 3953, 3953, 129, 3954, 3954, 130, 3956,+ 3956, 132, 3962, 3965, 130, 3968, 3968, 130, 3970, 3971, 230, 3972, 3972, 9, 3974, 3975, 230,+ 4038, 4038, 220, 4151, 4151, 7, 4153, 4154, 9, 4237, 4237, 220, 4957, 4959, 230, 5908, 5908, 9,+ 5940, 5940, 9, 6098, 6098, 9, 6109, 6109, 230, 6313, 6313, 228, 6457, 6457, 222, 6458, 6458,+ 230, 6459, 6459, 220, 6679, 6679, 230, 6680, 6680, 220, 6752, 6752, 9, 6773, 6780, 230, 6783,+ 6783, 220, 6832, 6836, 230, 6837, 6842, 220, 6843, 6844, 230, 6845, 6845, 220, 6964, 6964, 7,+ 6980, 6980, 9, 7019, 7019, 230, 7020, 7020, 220, 7021, 7027, 230, 7082, 7083, 9, 7142, 7142, 7,+ 7154, 7155, 9, 7223, 7223, 7, 7376, 7378, 230, 7380, 7380, 1, 7381, 7385, 220, 7386, 7387, 230,+ 7388, 7391, 220, 7392, 7392, 230, 7394, 7400, 1, 7405, 7405, 220, 7412, 7412, 230, 7416, 7417,+ 230, 7616, 7617, 230, 7618, 7618, 220, 7619, 7625, 230, 7626, 7626, 220, 7627, 7628, 230, 7629,+ 7629, 234, 7630, 7630, 214, 7631, 7631, 220, 7632, 7632, 202, 7633, 7669, 230, 7675, 7675, 230,+ 7676, 7676, 233, 7677, 7677, 220, 7678, 7678, 230, 7679, 7679, 220, 8400, 8401, 230, 8402,+ 8403, 1, 8404, 8407, 230, 8408, 8410, 1, 8411, 8412, 230, 8417, 8417, 230, 8421, 8422, 1, 8423,+ 8423, 230, 8424, 8424, 220, 8425, 8425, 230, 8426, 8427, 1, 8428, 8431, 220, 8432, 8432, 230,+ 11503, 11505, 230, 11647, 11647, 9, 11744, 11775, 230, 12330, 12330, 218, 12331, 12331, 228,+ 12332, 12332, 232, 12333, 12333, 222, 12334, 12335, 224, 12441, 12442, 8, 42607, 42607, 230,+ 42612, 42621, 230, 42654, 42655, 230, 42736, 42737, 230, 43014, 43014, 9, 43204, 43204, 9,+ 43232, 43249, 230, 43307, 43309, 220, 43347, 43347, 9, 43443, 43443, 7, 43456, 43456, 9, 43696,+ 43696, 230, 43698, 43699, 230, 43700, 43700, 220, 43703, 43704, 230, 43710, 43711, 230, 43713,+ 43713, 230, 43766, 43766, 9, 44013, 44013, 9, 64286, 64286, 26, 65056, 65062, 230, 65063,+ 65069, 220, 65070, 65071, 230, 66045, 66045, 220, 66272, 66272, 220, 66422, 66426, 230, 68109,+ 68109, 220, 68111, 68111, 230, 68152, 68152, 230, 68153, 68153, 1, 68154, 68154, 220, 68159,+ 68159, 9, 68325, 68325, 230, 68326, 68326, 220, 69702, 69702, 9, 69759, 69759, 9, 69817, 69817,+ 9, 69818, 69818, 7, 69888, 69890, 230, 69939, 69940, 9, 70003, 70003, 7, 70080, 70080, 9,+ 70090, 70090, 7, 70197, 70197, 9, 70198, 70198, 7, 70377, 70377, 7, 70378, 70378, 9, 70460,+ 70460, 7, 70477, 70477, 9, 70502, 70508, 230, 70512, 70516, 230, 70722, 70722, 9, 70726, 70726,+ 7, 70850, 70850, 9, 70851, 70851, 7, 71103, 71103, 9, 71104, 71104, 7, 71231, 71231, 9, 71350,+ 71350, 9, 71351, 71351, 7, 71467, 71467, 9, 72767, 72767, 9, 92912, 92916, 1, 92976, 92982,+ 230, 113822, 113822, 1, 119141, 119142, 216, 119143, 119145, 1, 119149, 119149, 226, 119150,+ 119154, 216, 119163, 119170, 220, 119173, 119177, 230, 119178, 119179, 220, 119210, 119213,+ 230, 119362, 119364, 230, 122880, 122886, 230, 122888, 122904, 230, 122907, 122913, 230,+ 122915, 122916, 230, 122918, 122922, 230, 125136, 125142, 220, 125252, 125257, 230, 125258,+ 125258, 7,+};++// composeKeys are (first<<32 | second) of each primary composition pair, sorted;+// composeValues is the composite.+inline constexpr uint64_t k_compose_keys[] = {+ 0x3c00000338ULL, 0x3d00000338ULL, 0x3e00000338ULL, 0x4100000300ULL, 0x4100000301ULL,+ 0x4100000302ULL, 0x4100000303ULL, 0x4100000304ULL, 0x4100000306ULL, 0x4100000307ULL,+ 0x4100000308ULL, 0x4100000309ULL, 0x410000030aULL, 0x410000030cULL, 0x410000030fULL,+ 0x4100000311ULL, 0x4100000323ULL, 0x4100000325ULL, 0x4100000328ULL, 0x4200000307ULL,+ 0x4200000323ULL, 0x4200000331ULL, 0x4300000301ULL, 0x4300000302ULL, 0x4300000307ULL,+ 0x430000030cULL, 0x4300000327ULL, 0x4400000307ULL, 0x440000030cULL, 0x4400000323ULL,+ 0x4400000327ULL, 0x440000032dULL, 0x4400000331ULL, 0x4500000300ULL, 0x4500000301ULL,+ 0x4500000302ULL, 0x4500000303ULL, 0x4500000304ULL, 0x4500000306ULL, 0x4500000307ULL,+ 0x4500000308ULL, 0x4500000309ULL, 0x450000030cULL, 0x450000030fULL, 0x4500000311ULL,+ 0x4500000323ULL, 0x4500000327ULL, 0x4500000328ULL, 0x450000032dULL, 0x4500000330ULL,+ 0x4600000307ULL, 0x4700000301ULL, 0x4700000302ULL, 0x4700000304ULL, 0x4700000306ULL,+ 0x4700000307ULL, 0x470000030cULL, 0x4700000327ULL, 0x4800000302ULL, 0x4800000307ULL,+ 0x4800000308ULL, 0x480000030cULL, 0x4800000323ULL, 0x4800000327ULL, 0x480000032eULL,+ 0x4900000300ULL, 0x4900000301ULL, 0x4900000302ULL, 0x4900000303ULL, 0x4900000304ULL,+ 0x4900000306ULL, 0x4900000307ULL, 0x4900000308ULL, 0x4900000309ULL, 0x490000030cULL,+ 0x490000030fULL, 0x4900000311ULL, 0x4900000323ULL, 0x4900000328ULL, 0x4900000330ULL,+ 0x4a00000302ULL, 0x4b00000301ULL, 0x4b0000030cULL, 0x4b00000323ULL, 0x4b00000327ULL,+ 0x4b00000331ULL, 0x4c00000301ULL, 0x4c0000030cULL, 0x4c00000323ULL, 0x4c00000327ULL,+ 0x4c0000032dULL, 0x4c00000331ULL, 0x4d00000301ULL, 0x4d00000307ULL, 0x4d00000323ULL,+ 0x4e00000300ULL, 0x4e00000301ULL, 0x4e00000303ULL, 0x4e00000307ULL, 0x4e0000030cULL,+ 0x4e00000323ULL, 0x4e00000327ULL, 0x4e0000032dULL, 0x4e00000331ULL, 0x4f00000300ULL,+ 0x4f00000301ULL, 0x4f00000302ULL, 0x4f00000303ULL, 0x4f00000304ULL, 0x4f00000306ULL,+ 0x4f00000307ULL, 0x4f00000308ULL, 0x4f00000309ULL, 0x4f0000030bULL, 0x4f0000030cULL,+ 0x4f0000030fULL, 0x4f00000311ULL, 0x4f0000031bULL, 0x4f00000323ULL, 0x4f00000328ULL,+ 0x5000000301ULL, 0x5000000307ULL, 0x5200000301ULL, 0x5200000307ULL, 0x520000030cULL,+ 0x520000030fULL, 0x5200000311ULL, 0x5200000323ULL, 0x5200000327ULL, 0x5200000331ULL,+ 0x5300000301ULL, 0x5300000302ULL, 0x5300000307ULL, 0x530000030cULL, 0x5300000323ULL,+ 0x5300000326ULL, 0x5300000327ULL, 0x5400000307ULL, 0x540000030cULL, 0x5400000323ULL,+ 0x5400000326ULL, 0x5400000327ULL, 0x540000032dULL, 0x5400000331ULL, 0x5500000300ULL,+ 0x5500000301ULL, 0x5500000302ULL, 0x5500000303ULL, 0x5500000304ULL, 0x5500000306ULL,+ 0x5500000308ULL, 0x5500000309ULL, 0x550000030aULL, 0x550000030bULL, 0x550000030cULL,+ 0x550000030fULL, 0x5500000311ULL, 0x550000031bULL, 0x5500000323ULL, 0x5500000324ULL,+ 0x5500000328ULL, 0x550000032dULL, 0x5500000330ULL, 0x5600000303ULL, 0x5600000323ULL,+ 0x5700000300ULL, 0x5700000301ULL, 0x5700000302ULL, 0x5700000307ULL, 0x5700000308ULL,+ 0x5700000323ULL, 0x5800000307ULL, 0x5800000308ULL, 0x5900000300ULL, 0x5900000301ULL,+ 0x5900000302ULL, 0x5900000303ULL, 0x5900000304ULL, 0x5900000307ULL, 0x5900000308ULL,+ 0x5900000309ULL, 0x5900000323ULL, 0x5a00000301ULL, 0x5a00000302ULL, 0x5a00000307ULL,+ 0x5a0000030cULL, 0x5a00000323ULL, 0x5a00000331ULL, 0x6100000300ULL, 0x6100000301ULL,+ 0x6100000302ULL, 0x6100000303ULL, 0x6100000304ULL, 0x6100000306ULL, 0x6100000307ULL,+ 0x6100000308ULL, 0x6100000309ULL, 0x610000030aULL, 0x610000030cULL, 0x610000030fULL,+ 0x6100000311ULL, 0x6100000323ULL, 0x6100000325ULL, 0x6100000328ULL, 0x6200000307ULL,+ 0x6200000323ULL, 0x6200000331ULL, 0x6300000301ULL, 0x6300000302ULL, 0x6300000307ULL,+ 0x630000030cULL, 0x6300000327ULL, 0x6400000307ULL, 0x640000030cULL, 0x6400000323ULL,+ 0x6400000327ULL, 0x640000032dULL, 0x6400000331ULL, 0x6500000300ULL, 0x6500000301ULL,+ 0x6500000302ULL, 0x6500000303ULL, 0x6500000304ULL, 0x6500000306ULL, 0x6500000307ULL,+ 0x6500000308ULL, 0x6500000309ULL, 0x650000030cULL, 0x650000030fULL, 0x6500000311ULL,+ 0x6500000323ULL, 0x6500000327ULL, 0x6500000328ULL, 0x650000032dULL, 0x6500000330ULL,+ 0x6600000307ULL, 0x6700000301ULL, 0x6700000302ULL, 0x6700000304ULL, 0x6700000306ULL,+ 0x6700000307ULL, 0x670000030cULL, 0x6700000327ULL, 0x6800000302ULL, 0x6800000307ULL,+ 0x6800000308ULL, 0x680000030cULL, 0x6800000323ULL, 0x6800000327ULL, 0x680000032eULL,+ 0x6800000331ULL, 0x6900000300ULL, 0x6900000301ULL, 0x6900000302ULL, 0x6900000303ULL,+ 0x6900000304ULL, 0x6900000306ULL, 0x6900000308ULL, 0x6900000309ULL, 0x690000030cULL,+ 0x690000030fULL, 0x6900000311ULL, 0x6900000323ULL, 0x6900000328ULL, 0x6900000330ULL,+ 0x6a00000302ULL, 0x6a0000030cULL, 0x6b00000301ULL, 0x6b0000030cULL, 0x6b00000323ULL,+ 0x6b00000327ULL, 0x6b00000331ULL, 0x6c00000301ULL, 0x6c0000030cULL, 0x6c00000323ULL,+ 0x6c00000327ULL, 0x6c0000032dULL, 0x6c00000331ULL, 0x6d00000301ULL, 0x6d00000307ULL,+ 0x6d00000323ULL, 0x6e00000300ULL, 0x6e00000301ULL, 0x6e00000303ULL, 0x6e00000307ULL,+ 0x6e0000030cULL, 0x6e00000323ULL, 0x6e00000327ULL, 0x6e0000032dULL, 0x6e00000331ULL,+ 0x6f00000300ULL, 0x6f00000301ULL, 0x6f00000302ULL, 0x6f00000303ULL, 0x6f00000304ULL,+ 0x6f00000306ULL, 0x6f00000307ULL, 0x6f00000308ULL, 0x6f00000309ULL, 0x6f0000030bULL,+ 0x6f0000030cULL, 0x6f0000030fULL, 0x6f00000311ULL, 0x6f0000031bULL, 0x6f00000323ULL,+ 0x6f00000328ULL, 0x7000000301ULL, 0x7000000307ULL, 0x7200000301ULL, 0x7200000307ULL,+ 0x720000030cULL, 0x720000030fULL, 0x7200000311ULL, 0x7200000323ULL, 0x7200000327ULL,+ 0x7200000331ULL, 0x7300000301ULL, 0x7300000302ULL, 0x7300000307ULL, 0x730000030cULL,+ 0x7300000323ULL, 0x7300000326ULL, 0x7300000327ULL, 0x7400000307ULL, 0x7400000308ULL,+ 0x740000030cULL, 0x7400000323ULL, 0x7400000326ULL, 0x7400000327ULL, 0x740000032dULL,+ 0x7400000331ULL, 0x7500000300ULL, 0x7500000301ULL, 0x7500000302ULL, 0x7500000303ULL,+ 0x7500000304ULL, 0x7500000306ULL, 0x7500000308ULL, 0x7500000309ULL, 0x750000030aULL,+ 0x750000030bULL, 0x750000030cULL, 0x750000030fULL, 0x7500000311ULL, 0x750000031bULL,+ 0x7500000323ULL, 0x7500000324ULL, 0x7500000328ULL, 0x750000032dULL, 0x7500000330ULL,+ 0x7600000303ULL, 0x7600000323ULL, 0x7700000300ULL, 0x7700000301ULL, 0x7700000302ULL,+ 0x7700000307ULL, 0x7700000308ULL, 0x770000030aULL, 0x7700000323ULL, 0x7800000307ULL,+ 0x7800000308ULL, 0x7900000300ULL, 0x7900000301ULL, 0x7900000302ULL, 0x7900000303ULL,+ 0x7900000304ULL, 0x7900000307ULL, 0x7900000308ULL, 0x7900000309ULL, 0x790000030aULL,+ 0x7900000323ULL, 0x7a00000301ULL, 0x7a00000302ULL, 0x7a00000307ULL, 0x7a0000030cULL,+ 0x7a00000323ULL, 0x7a00000331ULL, 0xa800000300ULL, 0xa800000301ULL, 0xa800000342ULL,+ 0xc200000300ULL, 0xc200000301ULL, 0xc200000303ULL, 0xc200000309ULL, 0xc400000304ULL,+ 0xc500000301ULL, 0xc600000301ULL, 0xc600000304ULL, 0xc700000301ULL, 0xca00000300ULL,+ 0xca00000301ULL, 0xca00000303ULL, 0xca00000309ULL, 0xcf00000301ULL, 0xd400000300ULL,+ 0xd400000301ULL, 0xd400000303ULL, 0xd400000309ULL, 0xd500000301ULL, 0xd500000304ULL,+ 0xd500000308ULL, 0xd600000304ULL, 0xd800000301ULL, 0xdc00000300ULL, 0xdc00000301ULL,+ 0xdc00000304ULL, 0xdc0000030cULL, 0xe200000300ULL, 0xe200000301ULL, 0xe200000303ULL,+ 0xe200000309ULL, 0xe400000304ULL, 0xe500000301ULL, 0xe600000301ULL, 0xe600000304ULL,+ 0xe700000301ULL, 0xea00000300ULL, 0xea00000301ULL, 0xea00000303ULL, 0xea00000309ULL,+ 0xef00000301ULL, 0xf400000300ULL, 0xf400000301ULL, 0xf400000303ULL, 0xf400000309ULL,+ 0xf500000301ULL, 0xf500000304ULL, 0xf500000308ULL, 0xf600000304ULL, 0xf800000301ULL,+ 0xfc00000300ULL, 0xfc00000301ULL, 0xfc00000304ULL, 0xfc0000030cULL, 0x10200000300ULL,+ 0x10200000301ULL, 0x10200000303ULL, 0x10200000309ULL, 0x10300000300ULL, 0x10300000301ULL,+ 0x10300000303ULL, 0x10300000309ULL, 0x11200000300ULL, 0x11200000301ULL, 0x11300000300ULL,+ 0x11300000301ULL, 0x14c00000300ULL, 0x14c00000301ULL, 0x14d00000300ULL, 0x14d00000301ULL,+ 0x15a00000307ULL, 0x15b00000307ULL, 0x16000000307ULL, 0x16100000307ULL, 0x16800000301ULL,+ 0x16900000301ULL, 0x16a00000308ULL, 0x16b00000308ULL, 0x17f00000307ULL, 0x1a000000300ULL,+ 0x1a000000301ULL, 0x1a000000303ULL, 0x1a000000309ULL, 0x1a000000323ULL, 0x1a100000300ULL,+ 0x1a100000301ULL, 0x1a100000303ULL, 0x1a100000309ULL, 0x1a100000323ULL, 0x1af00000300ULL,+ 0x1af00000301ULL, 0x1af00000303ULL, 0x1af00000309ULL, 0x1af00000323ULL, 0x1b000000300ULL,+ 0x1b000000301ULL, 0x1b000000303ULL, 0x1b000000309ULL, 0x1b000000323ULL, 0x1b70000030cULL,+ 0x1ea00000304ULL, 0x1eb00000304ULL, 0x22600000304ULL, 0x22700000304ULL, 0x22800000306ULL,+ 0x22900000306ULL, 0x22e00000304ULL, 0x22f00000304ULL, 0x2920000030cULL, 0x39100000300ULL,+ 0x39100000301ULL, 0x39100000304ULL, 0x39100000306ULL, 0x39100000313ULL, 0x39100000314ULL,+ 0x39100000345ULL, 0x39500000300ULL, 0x39500000301ULL, 0x39500000313ULL, 0x39500000314ULL,+ 0x39700000300ULL, 0x39700000301ULL, 0x39700000313ULL, 0x39700000314ULL, 0x39700000345ULL,+ 0x39900000300ULL, 0x39900000301ULL, 0x39900000304ULL, 0x39900000306ULL, 0x39900000308ULL,+ 0x39900000313ULL, 0x39900000314ULL, 0x39f00000300ULL, 0x39f00000301ULL, 0x39f00000313ULL,+ 0x39f00000314ULL, 0x3a100000314ULL, 0x3a500000300ULL, 0x3a500000301ULL, 0x3a500000304ULL,+ 0x3a500000306ULL, 0x3a500000308ULL, 0x3a500000314ULL, 0x3a900000300ULL, 0x3a900000301ULL,+ 0x3a900000313ULL, 0x3a900000314ULL, 0x3a900000345ULL, 0x3ac00000345ULL, 0x3ae00000345ULL,+ 0x3b100000300ULL, 0x3b100000301ULL, 0x3b100000304ULL, 0x3b100000306ULL, 0x3b100000313ULL,+ 0x3b100000314ULL, 0x3b100000342ULL, 0x3b100000345ULL, 0x3b500000300ULL, 0x3b500000301ULL,+ 0x3b500000313ULL, 0x3b500000314ULL, 0x3b700000300ULL, 0x3b700000301ULL, 0x3b700000313ULL,+ 0x3b700000314ULL, 0x3b700000342ULL, 0x3b700000345ULL, 0x3b900000300ULL, 0x3b900000301ULL,+ 0x3b900000304ULL, 0x3b900000306ULL, 0x3b900000308ULL, 0x3b900000313ULL, 0x3b900000314ULL,+ 0x3b900000342ULL, 0x3bf00000300ULL, 0x3bf00000301ULL, 0x3bf00000313ULL, 0x3bf00000314ULL,+ 0x3c100000313ULL, 0x3c100000314ULL, 0x3c500000300ULL, 0x3c500000301ULL, 0x3c500000304ULL,+ 0x3c500000306ULL, 0x3c500000308ULL, 0x3c500000313ULL, 0x3c500000314ULL, 0x3c500000342ULL,+ 0x3c900000300ULL, 0x3c900000301ULL, 0x3c900000313ULL, 0x3c900000314ULL, 0x3c900000342ULL,+ 0x3c900000345ULL, 0x3ca00000300ULL, 0x3ca00000301ULL, 0x3ca00000342ULL, 0x3cb00000300ULL,+ 0x3cb00000301ULL, 0x3cb00000342ULL, 0x3ce00000345ULL, 0x3d200000301ULL, 0x3d200000308ULL,+ 0x40600000308ULL, 0x41000000306ULL, 0x41000000308ULL, 0x41300000301ULL, 0x41500000300ULL,+ 0x41500000306ULL, 0x41500000308ULL, 0x41600000306ULL, 0x41600000308ULL, 0x41700000308ULL,+ 0x41800000300ULL, 0x41800000304ULL, 0x41800000306ULL, 0x41800000308ULL, 0x41a00000301ULL,+ 0x41e00000308ULL, 0x42300000304ULL, 0x42300000306ULL, 0x42300000308ULL, 0x4230000030bULL,+ 0x42700000308ULL, 0x42b00000308ULL, 0x42d00000308ULL, 0x43000000306ULL, 0x43000000308ULL,+ 0x43300000301ULL, 0x43500000300ULL, 0x43500000306ULL, 0x43500000308ULL, 0x43600000306ULL,+ 0x43600000308ULL, 0x43700000308ULL, 0x43800000300ULL, 0x43800000304ULL, 0x43800000306ULL,+ 0x43800000308ULL, 0x43a00000301ULL, 0x43e00000308ULL, 0x44300000304ULL, 0x44300000306ULL,+ 0x44300000308ULL, 0x4430000030bULL, 0x44700000308ULL, 0x44b00000308ULL, 0x44d00000308ULL,+ 0x45600000308ULL, 0x4740000030fULL, 0x4750000030fULL, 0x4d800000308ULL, 0x4d900000308ULL,+ 0x4e800000308ULL, 0x4e900000308ULL, 0x62700000653ULL, 0x62700000654ULL, 0x62700000655ULL,+ 0x64800000654ULL, 0x64a00000654ULL, 0x6c100000654ULL, 0x6d200000654ULL, 0x6d500000654ULL,+ 0x9280000093cULL, 0x9300000093cULL, 0x9330000093cULL, 0x9c7000009beULL, 0x9c7000009d7ULL,+ 0xb4700000b3eULL, 0xb4700000b56ULL, 0xb4700000b57ULL, 0xb9200000bd7ULL, 0xbc600000bbeULL,+ 0xbc600000bd7ULL, 0xbc700000bbeULL, 0xc4600000c56ULL, 0xcbf00000cd5ULL, 0xcc600000cc2ULL,+ 0xcc600000cd5ULL, 0xcc600000cd6ULL, 0xcca00000cd5ULL, 0xd4600000d3eULL, 0xd4600000d57ULL,+ 0xd4700000d3eULL, 0xdd900000dcaULL, 0xdd900000dcfULL, 0xdd900000ddfULL, 0xddc00000dcaULL,+ 0x10250000102eULL, 0x1b0500001b35ULL, 0x1b0700001b35ULL, 0x1b0900001b35ULL, 0x1b0b00001b35ULL,+ 0x1b0d00001b35ULL, 0x1b1100001b35ULL, 0x1b3a00001b35ULL, 0x1b3c00001b35ULL, 0x1b3e00001b35ULL,+ 0x1b3f00001b35ULL, 0x1b4200001b35ULL, 0x1e3600000304ULL, 0x1e3700000304ULL, 0x1e5a00000304ULL,+ 0x1e5b00000304ULL, 0x1e6200000307ULL, 0x1e6300000307ULL, 0x1ea000000302ULL, 0x1ea000000306ULL,+ 0x1ea100000302ULL, 0x1ea100000306ULL, 0x1eb800000302ULL, 0x1eb900000302ULL, 0x1ecc00000302ULL,+ 0x1ecd00000302ULL, 0x1f0000000300ULL, 0x1f0000000301ULL, 0x1f0000000342ULL, 0x1f0000000345ULL,+ 0x1f0100000300ULL, 0x1f0100000301ULL, 0x1f0100000342ULL, 0x1f0100000345ULL, 0x1f0200000345ULL,+ 0x1f0300000345ULL, 0x1f0400000345ULL, 0x1f0500000345ULL, 0x1f0600000345ULL, 0x1f0700000345ULL,+ 0x1f0800000300ULL, 0x1f0800000301ULL, 0x1f0800000342ULL, 0x1f0800000345ULL, 0x1f0900000300ULL,+ 0x1f0900000301ULL, 0x1f0900000342ULL, 0x1f0900000345ULL, 0x1f0a00000345ULL, 0x1f0b00000345ULL,+ 0x1f0c00000345ULL, 0x1f0d00000345ULL, 0x1f0e00000345ULL, 0x1f0f00000345ULL, 0x1f1000000300ULL,+ 0x1f1000000301ULL, 0x1f1100000300ULL, 0x1f1100000301ULL, 0x1f1800000300ULL, 0x1f1800000301ULL,+ 0x1f1900000300ULL, 0x1f1900000301ULL, 0x1f2000000300ULL, 0x1f2000000301ULL, 0x1f2000000342ULL,+ 0x1f2000000345ULL, 0x1f2100000300ULL, 0x1f2100000301ULL, 0x1f2100000342ULL, 0x1f2100000345ULL,+ 0x1f2200000345ULL, 0x1f2300000345ULL, 0x1f2400000345ULL, 0x1f2500000345ULL, 0x1f2600000345ULL,+ 0x1f2700000345ULL, 0x1f2800000300ULL, 0x1f2800000301ULL, 0x1f2800000342ULL, 0x1f2800000345ULL,+ 0x1f2900000300ULL, 0x1f2900000301ULL, 0x1f2900000342ULL, 0x1f2900000345ULL, 0x1f2a00000345ULL,+ 0x1f2b00000345ULL, 0x1f2c00000345ULL, 0x1f2d00000345ULL, 0x1f2e00000345ULL, 0x1f2f00000345ULL,+ 0x1f3000000300ULL, 0x1f3000000301ULL, 0x1f3000000342ULL, 0x1f3100000300ULL, 0x1f3100000301ULL,+ 0x1f3100000342ULL, 0x1f3800000300ULL, 0x1f3800000301ULL, 0x1f3800000342ULL, 0x1f3900000300ULL,+ 0x1f3900000301ULL, 0x1f3900000342ULL, 0x1f4000000300ULL, 0x1f4000000301ULL, 0x1f4100000300ULL,+ 0x1f4100000301ULL, 0x1f4800000300ULL, 0x1f4800000301ULL, 0x1f4900000300ULL, 0x1f4900000301ULL,+ 0x1f5000000300ULL, 0x1f5000000301ULL, 0x1f5000000342ULL, 0x1f5100000300ULL, 0x1f5100000301ULL,+ 0x1f5100000342ULL, 0x1f5900000300ULL, 0x1f5900000301ULL, 0x1f5900000342ULL, 0x1f6000000300ULL,+ 0x1f6000000301ULL, 0x1f6000000342ULL, 0x1f6000000345ULL, 0x1f6100000300ULL, 0x1f6100000301ULL,+ 0x1f6100000342ULL, 0x1f6100000345ULL, 0x1f6200000345ULL, 0x1f6300000345ULL, 0x1f6400000345ULL,+ 0x1f6500000345ULL, 0x1f6600000345ULL, 0x1f6700000345ULL, 0x1f6800000300ULL, 0x1f6800000301ULL,+ 0x1f6800000342ULL, 0x1f6800000345ULL, 0x1f6900000300ULL, 0x1f6900000301ULL, 0x1f6900000342ULL,+ 0x1f6900000345ULL, 0x1f6a00000345ULL, 0x1f6b00000345ULL, 0x1f6c00000345ULL, 0x1f6d00000345ULL,+ 0x1f6e00000345ULL, 0x1f6f00000345ULL, 0x1f7000000345ULL, 0x1f7400000345ULL, 0x1f7c00000345ULL,+ 0x1fb600000345ULL, 0x1fbf00000300ULL, 0x1fbf00000301ULL, 0x1fbf00000342ULL, 0x1fc600000345ULL,+ 0x1ff600000345ULL, 0x1ffe00000300ULL, 0x1ffe00000301ULL, 0x1ffe00000342ULL, 0x219000000338ULL,+ 0x219200000338ULL, 0x219400000338ULL, 0x21d000000338ULL, 0x21d200000338ULL, 0x21d400000338ULL,+ 0x220300000338ULL, 0x220800000338ULL, 0x220b00000338ULL, 0x222300000338ULL, 0x222500000338ULL,+ 0x223c00000338ULL, 0x224300000338ULL, 0x224500000338ULL, 0x224800000338ULL, 0x224d00000338ULL,+ 0x226100000338ULL, 0x226400000338ULL, 0x226500000338ULL, 0x227200000338ULL, 0x227300000338ULL,+ 0x227600000338ULL, 0x227700000338ULL, 0x227a00000338ULL, 0x227b00000338ULL, 0x227c00000338ULL,+ 0x227d00000338ULL, 0x228200000338ULL, 0x228300000338ULL, 0x228600000338ULL, 0x228700000338ULL,+ 0x229100000338ULL, 0x229200000338ULL, 0x22a200000338ULL, 0x22a800000338ULL, 0x22a900000338ULL,+ 0x22ab00000338ULL, 0x22b200000338ULL, 0x22b300000338ULL, 0x22b400000338ULL, 0x22b500000338ULL,+ 0x304600003099ULL, 0x304b00003099ULL, 0x304d00003099ULL, 0x304f00003099ULL, 0x305100003099ULL,+ 0x305300003099ULL, 0x305500003099ULL, 0x305700003099ULL, 0x305900003099ULL, 0x305b00003099ULL,+ 0x305d00003099ULL, 0x305f00003099ULL, 0x306100003099ULL, 0x306400003099ULL, 0x306600003099ULL,+ 0x306800003099ULL, 0x306f00003099ULL, 0x306f0000309aULL, 0x307200003099ULL, 0x30720000309aULL,+ 0x307500003099ULL, 0x30750000309aULL, 0x307800003099ULL, 0x30780000309aULL, 0x307b00003099ULL,+ 0x307b0000309aULL, 0x309d00003099ULL, 0x30a600003099ULL, 0x30ab00003099ULL, 0x30ad00003099ULL,+ 0x30af00003099ULL, 0x30b100003099ULL, 0x30b300003099ULL, 0x30b500003099ULL, 0x30b700003099ULL,+ 0x30b900003099ULL, 0x30bb00003099ULL, 0x30bd00003099ULL, 0x30bf00003099ULL, 0x30c100003099ULL,+ 0x30c400003099ULL, 0x30c600003099ULL, 0x30c800003099ULL, 0x30cf00003099ULL, 0x30cf0000309aULL,+ 0x30d200003099ULL, 0x30d20000309aULL, 0x30d500003099ULL, 0x30d50000309aULL, 0x30d800003099ULL,+ 0x30d80000309aULL, 0x30db00003099ULL, 0x30db0000309aULL, 0x30ef00003099ULL, 0x30f000003099ULL,+ 0x30f100003099ULL, 0x30f200003099ULL, 0x30fd00003099ULL, 0x11099000110baULL,+ 0x1109b000110baULL, 0x110a5000110baULL, 0x1113100011127ULL, 0x1113200011127ULL,+ 0x113470001133eULL, 0x1134700011357ULL, 0x114b9000114b0ULL, 0x114b9000114baULL,+ 0x114b9000114bdULL, 0x115b8000115afULL, 0x115b9000115afULL,+};++// the composite of k_compose_keys+inline constexpr uint32_t k_compose_values[] = {+ 8814, 8800, 8815, 192, 193, 194, 195, 256, 258, 550, 196, 7842, 197, 461, 512, 514, 7840, 7680,+ 260, 7682, 7684, 7686, 262, 264, 266, 268, 199, 7690, 270, 7692, 7696, 7698, 7694, 200, 201,+ 202, 7868, 274, 276, 278, 203, 7866, 282, 516, 518, 7864, 552, 280, 7704, 7706, 7710, 500, 284,+ 7712, 286, 288, 486, 290, 292, 7714, 7718, 542, 7716, 7720, 7722, 204, 205, 206, 296, 298, 300,+ 304, 207, 7880, 463, 520, 522, 7882, 302, 7724, 308, 7728, 488, 7730, 310, 7732, 313, 317,+ 7734, 315, 7740, 7738, 7742, 7744, 7746, 504, 323, 209, 7748, 327, 7750, 325, 7754, 7752, 210,+ 211, 212, 213, 332, 334, 558, 214, 7886, 336, 465, 524, 526, 416, 7884, 490, 7764, 7766, 340,+ 7768, 344, 528, 530, 7770, 342, 7774, 346, 348, 7776, 352, 7778, 536, 350, 7786, 356, 7788,+ 538, 354, 7792, 7790, 217, 218, 219, 360, 362, 364, 220, 7910, 366, 368, 467, 532, 534, 431,+ 7908, 7794, 370, 7798, 7796, 7804, 7806, 7808, 7810, 372, 7814, 7812, 7816, 7818, 7820, 7922,+ 221, 374, 7928, 562, 7822, 376, 7926, 7924, 377, 7824, 379, 381, 7826, 7828, 224, 225, 226,+ 227, 257, 259, 551, 228, 7843, 229, 462, 513, 515, 7841, 7681, 261, 7683, 7685, 7687, 263, 265,+ 267, 269, 231, 7691, 271, 7693, 7697, 7699, 7695, 232, 233, 234, 7869, 275, 277, 279, 235,+ 7867, 283, 517, 519, 7865, 553, 281, 7705, 7707, 7711, 501, 285, 7713, 287, 289, 487, 291, 293,+ 7715, 7719, 543, 7717, 7721, 7723, 7830, 236, 237, 238, 297, 299, 301, 239, 7881, 464, 521,+ 523, 7883, 303, 7725, 309, 496, 7729, 489, 7731, 311, 7733, 314, 318, 7735, 316, 7741, 7739,+ 7743, 7745, 7747, 505, 324, 241, 7749, 328, 7751, 326, 7755, 7753, 242, 243, 244, 245, 333,+ 335, 559, 246, 7887, 337, 466, 525, 527, 417, 7885, 491, 7765, 7767, 341, 7769, 345, 529, 531,+ 7771, 343, 7775, 347, 349, 7777, 353, 7779, 537, 351, 7787, 7831, 357, 7789, 539, 355, 7793,+ 7791, 249, 250, 251, 361, 363, 365, 252, 7911, 367, 369, 468, 533, 535, 432, 7909, 7795, 371,+ 7799, 7797, 7805, 7807, 7809, 7811, 373, 7815, 7813, 7832, 7817, 7819, 7821, 7923, 253, 375,+ 7929, 563, 7823, 255, 7927, 7833, 7925, 378, 7825, 380, 382, 7827, 7829, 8173, 901, 8129, 7846,+ 7844, 7850, 7848, 478, 506, 508, 482, 7688, 7872, 7870, 7876, 7874, 7726, 7890, 7888, 7894,+ 7892, 7756, 556, 7758, 554, 510, 475, 471, 469, 473, 7847, 7845, 7851, 7849, 479, 507, 509,+ 483, 7689, 7873, 7871, 7877, 7875, 7727, 7891, 7889, 7895, 7893, 7757, 557, 7759, 555, 511,+ 476, 472, 470, 474, 7856, 7854, 7860, 7858, 7857, 7855, 7861, 7859, 7700, 7702, 7701, 7703,+ 7760, 7762, 7761, 7763, 7780, 7781, 7782, 7783, 7800, 7801, 7802, 7803, 7835, 7900, 7898, 7904,+ 7902, 7906, 7901, 7899, 7905, 7903, 7907, 7914, 7912, 7918, 7916, 7920, 7915, 7913, 7919, 7917,+ 7921, 494, 492, 493, 480, 481, 7708, 7709, 560, 561, 495, 8122, 902, 8121, 8120, 7944, 7945,+ 8124, 8136, 904, 7960, 7961, 8138, 905, 7976, 7977, 8140, 8154, 906, 8153, 8152, 938, 7992,+ 7993, 8184, 908, 8008, 8009, 8172, 8170, 910, 8169, 8168, 939, 8025, 8186, 911, 8040, 8041,+ 8188, 8116, 8132, 8048, 940, 8113, 8112, 7936, 7937, 8118, 8115, 8050, 941, 7952, 7953, 8052,+ 942, 7968, 7969, 8134, 8131, 8054, 943, 8145, 8144, 970, 7984, 7985, 8150, 8056, 972, 8000,+ 8001, 8164, 8165, 8058, 973, 8161, 8160, 971, 8016, 8017, 8166, 8060, 974, 8032, 8033, 8182,+ 8179, 8146, 912, 8151, 8162, 944, 8167, 8180, 979, 980, 1031, 1232, 1234, 1027, 1024, 1238,+ 1025, 1217, 1244, 1246, 1037, 1250, 1049, 1252, 1036, 1254, 1262, 1038, 1264, 1266, 1268, 1272,+ 1260, 1233, 1235, 1107, 1104, 1239, 1105, 1218, 1245, 1247, 1117, 1251, 1081, 1253, 1116, 1255,+ 1263, 1118, 1265, 1267, 1269, 1273, 1261, 1111, 1142, 1143, 1242, 1243, 1258, 1259, 1570, 1571,+ 1573, 1572, 1574, 1730, 1747, 1728, 2345, 2353, 2356, 2507, 2508, 2891, 2888, 2892, 2964, 3018,+ 3020, 3019, 3144, 3264, 3274, 3271, 3272, 3275, 3402, 3404, 3403, 3546, 3548, 3550, 3549, 4134,+ 6918, 6920, 6922, 6924, 6926, 6930, 6971, 6973, 6976, 6977, 6979, 7736, 7737, 7772, 7773, 7784,+ 7785, 7852, 7862, 7853, 7863, 7878, 7879, 7896, 7897, 7938, 7940, 7942, 8064, 7939, 7941, 7943,+ 8065, 8066, 8067, 8068, 8069, 8070, 8071, 7946, 7948, 7950, 8072, 7947, 7949, 7951, 8073, 8074,+ 8075, 8076, 8077, 8078, 8079, 7954, 7956, 7955, 7957, 7962, 7964, 7963, 7965, 7970, 7972, 7974,+ 8080, 7971, 7973, 7975, 8081, 8082, 8083, 8084, 8085, 8086, 8087, 7978, 7980, 7982, 8088, 7979,+ 7981, 7983, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 7986, 7988, 7990, 7987, 7989, 7991, 7994,+ 7996, 7998, 7995, 7997, 7999, 8002, 8004, 8003, 8005, 8010, 8012, 8011, 8013, 8018, 8020, 8022,+ 8019, 8021, 8023, 8027, 8029, 8031, 8034, 8036, 8038, 8096, 8035, 8037, 8039, 8097, 8098, 8099,+ 8100, 8101, 8102, 8103, 8042, 8044, 8046, 8104, 8043, 8045, 8047, 8105, 8106, 8107, 8108, 8109,+ 8110, 8111, 8114, 8130, 8178, 8119, 8141, 8142, 8143, 8135, 8183, 8157, 8158, 8159, 8602, 8603,+ 8622, 8653, 8655, 8654, 8708, 8713, 8716, 8740, 8742, 8769, 8772, 8775, 8777, 8813, 8802, 8816,+ 8817, 8820, 8821, 8824, 8825, 8832, 8833, 8928, 8929, 8836, 8837, 8840, 8841, 8930, 8931, 8876,+ 8877, 8878, 8879, 8938, 8939, 8940, 8941, 12436, 12364, 12366, 12368, 12370, 12372, 12374,+ 12376, 12378, 12380, 12382, 12384, 12386, 12389, 12391, 12393, 12400, 12401, 12403, 12404,+ 12406, 12407, 12409, 12410, 12412, 12413, 12446, 12532, 12460, 12462, 12464, 12466, 12468,+ 12470, 12472, 12474, 12476, 12478, 12480, 12482, 12485, 12487, 12489, 12496, 12497, 12499,+ 12500, 12502, 12503, 12505, 12506, 12508, 12509, 12535, 12536, 12537, 12538, 12542, 69786,+ 69788, 69803, 69934, 69935, 70475, 70476, 70844, 70843, 70846, 71098, 71099,+};diff --git a/laya-cpp/tests/tokenizer_test.cpp b/laya-cpp/tests/tokenizer_test.cppnew file mode 100644index 00000000..d29ce37e--- /dev/null+++ b/laya-cpp/tests/tokenizer_test.cpp@@ -0,0 +1,636 @@+// laya::tokenizer against the `tokenizers` crate (through the Python-recorded goldens) and against+// hand-built tokenizers. Ported from laya-go/tokenizer's tests.+#include <cstdlib>+#include <fstream>+#include <mutex>+#include <thread>++#include "golden.hpp"+#include "laya/pyvalue.hpp"+#include "laya/sha256.hpp"+#include "laya/tokenizer.hpp"+#include "laya/unicodedata.hpp"++using laya::tokenizer::Form;+using laya::tokenizer::Tokenizer;+using laya_test::At;+using laya_test::Value;+namespace tk = laya::tokenizer;+namespace ud = laya::unicodedata;+namespace pv = laya::pyvalue;++namespace {++const char* const kShipped[] = {"english", "multilingual"};++std::vector<int32_t> Ints(const Value& v) {+ std::vector<int32_t> out;+ for (const Value& x : v.items()) out.push_back(int32_t(x.as_int()));+ return out;+}+std::string Join(const std::vector<std::string>& v) {+ std::string out;+ for (const auto& s : v) out += s + "\x01";+ return out;+}+std::string Ids(const std::vector<int32_t>& v) {+ std::string out = "[";+ for (size_t i = 0; i < v.size() && i < 24; i++) out += std::to_string(v[i]) + " ";+ return out + (v.size() > 24 ? "... " : "") + "(" + std::to_string(v.size()) + ")]";+}+std::string Hex(std::string_view s) {+ std::string out;+ for (unsigned char c : s) {+ char b[4];+ std::snprintf(b, sizeof b, "%02x", c);+ out += b;+ }+ return out;+}++// A shipped checkpoint's tokenizer, loaded once. Without LAYA_CHECKPOINTS the test is skipped+// locally and FAILS under LAYA_REQUIRE_GOLDENS=1 (CI), so a missing checkpoint cannot pass as green.+const Tokenizer* Checkpoint(const std::string& name) {+ static std::mutex mu;+ static std::map<std::string, Tokenizer> loaded;+ std::lock_guard<std::mutex> lock(mu);+ auto it = loaded.find(name);+ if (it != loaded.end()) return &it->second;+ const char* root = std::getenv("LAYA_CHECKPOINTS");+ if (root == nullptr || *root == 0) return nullptr;+ auto t = Tokenizer::LoadDir(std::string(root) + "/" + name);+ if (!t.ok()) {+ ADD_FAILURE() << "load the " << name << " tokenizer: " << t.error();+ return nullptr;+ }+ return &loaded.emplace(name, std::move(t).value()).first->second;+}++#define LAYA_CHECKPOINT(var, name) \+ const Tokenizer* var = Checkpoint(name); \+ if (var == nullptr) { \+ const char* req = std::getenv("LAYA_REQUIRE_GOLDENS"); \+ if (req != nullptr && std::string(req) == "1" && !HasFailure()) { \+ ADD_FAILURE() << "LAYA_CHECKPOINTS is unset but LAYA_REQUIRE_GOLDENS=1"; \+ } else if (!HasFailure()) { \+ GTEST_SKIP() << "LAYA_CHECKPOINTS unset (local run)"; \+ } \+ return; \+ }++Tokenizer MustParse(const std::string& json, const char* config = nullptr) {+ auto t = Tokenizer::Parse(json, config == nullptr ? std::nullopt : std::optional<std::string_view>(config));+ EXPECT_TRUE(t.ok()) << (t.ok() ? "" : t.error());+ if (!t.ok()) std::abort();+ return std::move(t).value();+}++// A tiny byte-level tokenizer built in the test, so the loader is checked without a checkpoint.+std::string ByteLevelSpec(const std::string& extra) {+ std::string vocab;+ for (int b = 0, lifted = 0; b < 256; b++) {+ char32_t r;+ if ((b >= '!' && b <= '~') || (b >= 0xA1 && b <= 0xAC) || (b >= 0xAE && b <= 0xFF)) r = char32_t(b);+ else r = char32_t(256 + lifted++);+ std::string key;+ ud::AppendRune(key, r);+ if (r == '"' || r == '\\') key = "\\" + key;+ vocab += "\"" + key + "\":" + std::to_string(b) + ",";+ }+ vocab += "\"ab\":256";+ return "{\"model\":{\"type\":\"BPE\",\"vocab\":{" + vocab + "},\"merges\":[\"a b\"]},\"normalizer\":null," ++ "\"pre_tokenizer\":{\"type\":\"ByteLevel\",\"add_prefix_space\":false,\"use_regex\":true}" + extra + "}";+}+std::string Replace(std::string s, const std::string& from, const std::string& to) {+ size_t at = s.find(from);+ if (at != std::string::npos) s.replace(at, from.size(), to);+ return s;+}++std::string Digest(const std::function<void(const std::function<void(const std::string&)>&)>& rows) {+ laya::Sha256 h;+ rows([&](const std::string& s) { h.Update(s); });+ return laya::ToHex(h.Finish());+}+std::string Mapped(char32_t cp, const std::string& text) {+ std::string out = std::to_string(uint32_t(cp)) + ":";+ bool first = true;+ std::string_view rest(text);+ while (!rest.empty()) {+ auto d = ud::DecodeRune(rest);+ out += (first ? "" : ",") + std::to_string(uint32_t(d.rune));+ first = false;+ rest.remove_prefix(d.width);+ }+ return out + ";";+}+std::string One(char32_t cp) {+ std::string s;+ ud::AppendRune(s, cp);+ return s;+}+const char* PyBool(bool b) { return b ? "True" : "False"; }++} // namespace++// ---- normalization, lowercase and the single_word flank over every code point ----------------------+class FormSweep : public ::testing::TestWithParam<std::pair<const char*, Form>> {};+TEST_P(FormSweep, MatchesTheCrateOverAllUnicode) {+ LAYA_GOLDEN_LIST(sweep, "go_extra.json", "normalize", "sweep");+ Form form = GetParam().second;+ std::string got = Digest([&](const auto& yield) {+ for (char32_t r = 0; r < 0x110000; r++) {+ if (r >= 0xD800 && r <= 0xDFFF) continue;+ yield(Mapped(r, tk::Normalize(One(r), form)));+ }+ });+ EXPECT_EQ(got, sweep->Find(GetParam().first)->as_string()) << "some single code point differs from the crate";+}+INSTANTIATE_TEST_SUITE_P(Golden, FormSweep,+ ::testing::Values(std::make_pair("NFC", Form::NFC), std::make_pair("NFD", Form::NFD),+ std::make_pair("NFKC", Form::NFKC), std::make_pair("NFKD", Form::NFKD)));++TEST(TokenizerNormalize, RandomStringsMatchTheCrate) {+ LAYA_GOLDEN_LIST(cases, "go_extra.json", "normalize", "cases");+ ASSERT_GE(cases->size(), 1000u);+ for (size_t i = 0; i < cases->size(); i++) {+ const Value& c = cases->value(i);+ const std::string& text = c.Find("text")->as_string();+ const std::pair<const char*, Form> forms[] = {{"nfc", Form::NFC}, {"nfd", Form::NFD}, {"nfkc", Form::NFKC}, {"nfkd", Form::NFKD}};+ for (const auto& f : forms) {+ ASSERT_EQ(tk::Normalize(text, f.second), c.Find(f.first)->as_string()) << "case " << i << " " << f.first << " of " << Hex(text);+ }+ }+}++TEST(TokenizerNormalize, LowercaseMatchesTheCrateOverAllUnicode) {+ LAYA_GOLDEN_LIST(want, "go_extra.json", "lowercase", "sweep");+ std::string got = Digest([&](const auto& yield) {+ for (char32_t r = 0; r < 0x110000; r++) {+ if (r >= 0xD800 && r <= 0xDFFF) continue;+ yield(Mapped(r, tk::Lowercase(One(r))));+ }+ });+ EXPECT_EQ(got, want->as_string());+ EXPECT_EQ(tk::Lowercase("\xCE\xA3"), "\xCF\x83") << "no final-sigma rule: the crate lowercases per character";+}++// A single_word added token is refused beside a word character. Which characters those are is the+// crate's rule, not an alphanumeric test: it counts combining marks and joiners and not other numbers.+TEST(TokenizerAdded, SingleWordFlankMatchesTheCrate) {+ LAYA_GOLDEN_LIST(sw, "go_extra.json", "single_word");+ Tokenizer tok = MustParse(+ R"({"model":{"type":"BPE","vocab":{"a":0,"<unk>":2},"merges":[],"unk_token":"<unk>"},+ "added_tokens":[{"content":"TOK","id":1,"single_word":true,"special":true}]})");+ auto standalone = [&](const std::string& s) {+ for (int32_t id : tok.Encode(s)) {+ if (id == 1) return std::string("1");+ }+ return std::string("0");+ };+ for (const char* side : {"left_standalone", "right_standalone"}) {+ bool left = std::string(side) == "left_standalone";+ std::string got = Digest([&](const auto& yield) {+ for (char32_t r = 0; r < 0x110000; r++) {+ if (r >= 0xD800 && r <= 0xDFFF) {+ yield("0");+ continue;+ }+ yield(standalone(left ? One(r) + "TOK" : "TOK" + One(r)));+ }+ });+ EXPECT_EQ(got, sw->Find(side)->as_string()) << side;+ }+}++// ---- pre-tokenizers and normalizer specs ---------------------------------------------------------------+TEST(TokenizerPre, PreTokenizersMatchTheCrate) {+ LAYA_GOLDEN_LIST(pre, "go_extra.json", "pretokenize");+ const Value& texts = *pre->Find("texts");+ size_t checked = 0;+ auto check = [&](const std::string& name, const std::string& spec, const Value& want) {+ ASSERT_EQ(want.size(), texts.size()) << name;+ for (size_t i = 0; i < texts.size(); i++) {+ auto got = tk::RunPreTokenizerSpec(spec, texts.value(i).as_string());+ ASSERT_TRUE(got.ok()) << got.error();+ std::vector<std::string> w;+ for (const Value& x : want.value(i).items()) w.push_back(x.as_string());+ ASSERT_EQ(Join(got.value()), Join(w)) << name << " of " << Hex(texts.value(i).as_string());+ checked++;+ }+ };+ for (bool prefix : {false, true}) {+ for (bool regex : {true, false}) {+ std::string key = std::string("add_prefix_space=") + PyBool(prefix) + ",use_regex=" + PyBool(regex);+ check("ByteLevel/" + key,+ std::string("{\"type\":\"ByteLevel\",\"add_prefix_space\":") + (prefix ? "true" : "false") + ",\"use_regex\":" + (regex ? "true" : "false") + "}",+ *At(pre, {"byte_level", key.c_str()}));+ }+ }+ for (const char* scheme : {"always", "first", "never"}) {+ for (bool split : {true, false}) {+ std::string key = std::string(scheme) + ",split=" + PyBool(split);+ check("Metaspace/" + key,+ std::string("{\"type\":\"Metaspace\",\"replacement\":\"\xE2\x96\x81\",\"prepend_scheme\":\"") + scheme + "\",\"split\":" + (split ? "true" : "false") + "}",+ *At(pre, {"metaspace", key.c_str()}));+ }+ }+ EXPECT_GE(checked, 10u * 1000u);+}++TEST(TokenizerNormalize, NormalizerSpecsMatchTheCrate) {+ LAYA_GOLDEN_LIST(fam, "go_extra.json", "normalizers");+ const Value& inputs = *fam->Find("inputs");+ const Value& specs = *fam->Find("specs");+ ASSERT_GE(specs.size(), 12u);+ for (size_t s = 0; s < specs.size(); s++) {+ auto spec = pv::Dumps(*specs.value(s).Find("spec"));+ ASSERT_TRUE(spec.ok());+ const Value& outs = *specs.value(s).Find("outputs");+ for (size_t j = 0; j < inputs.size(); j++) {+ auto got = tk::RunNormalizerSpec(spec.value(), inputs.value(j).as_string());+ ASSERT_TRUE(got.ok()) << spec.value() << ": " << got.error();+ ASSERT_EQ(got.value(), outs.value(j).as_string()) << spec.value() << " on " << Hex(inputs.value(j).as_string());+ }+ }+}++// ---- the shipped tokenizers against Python ------------------------------------------------------------+class Shipped : public ::testing::TestWithParam<const char*> {};+INSTANTIATE_TEST_SUITE_P(Checkpoint, Shipped, ::testing::ValuesIn(kShipped));++TEST_P(Shipped, CorpusIdsIdentityAndSpecialsMatchPython) {+ std::string name = GetParam();+ LAYA_GOLDEN_LIST(side, "tokenizer_ids.json", name.c_str());+ LAYA_CHECKPOINT(tok, name);+ const Value* corpus_text = At(laya_test::Load("tokenizer_ids.json", true), {"corpus_text"});+ ASSERT_NE(corpus_text, nullptr);+ const Value& id = *side->Find("identity");+ size_t want_vocab = size_t(id.Find("vocab_size")->as_int());+ if (const Value* extra = id.Find("added_tokens_outside_vocab"); extra != nullptr && !extra->is_null()) want_vocab += size_t(extra->as_int());+ EXPECT_EQ(tok->VocabSize(), want_vocab);+ const Value& sp = *side->Find("special_ids");+ EXPECT_EQ(tok->Cls(), std::optional<int32_t>(int32_t(sp.Find("cls")->as_int())));+ EXPECT_EQ(tok->Sep(), std::optional<int32_t>(int32_t(sp.Find("sep")->as_int())));+ EXPECT_EQ(tok->Mask(), std::optional<int32_t>(int32_t(sp.Find("mask")->as_int())));+ EXPECT_EQ(tok->Pad(), std::optional<int32_t>(int32_t(sp.Find("pad")->as_int())));+ EXPECT_EQ(tok->MaskToken(), sp.Find("mask_token")->as_string());+ const Value& corpus = *side->Find("corpus");+ ASSERT_GE(corpus.size(), 35u);+ for (size_t i = 0; i < corpus.size(); i++) {+ const std::string& text = corpus_text->Find(corpus.key(i))->as_string();+ auto got = tok->Encode(text);+ EXPECT_EQ(got, Ints(corpus.value(i))) << corpus.key(i) << " " << Ids(got);+ }+ const Value& trunc = *side->Find("option_truncation");+ std::string text = " " + trunc.Find("text")->as_string();+ EXPECT_EQ(tok->Encode(text), Ints(*trunc.Find("uncapped")));+ EXPECT_EQ(tok->Encode(text, 48), Ints(*trunc.Find("capped_48")));+}++TEST_P(Shipped, DecodeMatchesPythonWholeAndSliced) {+ std::string name = GetParam();+ LAYA_GOLDEN_LIST(entry, "decode_text.json", "by", name.c_str());+ LAYA_CHECKPOINT(tok, name);+ const Value& cases = *entry->Find("cases");+ ASSERT_GE(cases.size(), 23u);+ size_t slices = 0;+ for (const Value& c : cases.items()) {+ const std::string& text = c.Find("text")->as_string();+ auto ids = Ints(*c.Find("ids"));+ EXPECT_EQ(tok->Encode(text), ids) << Hex(text);+ EXPECT_EQ(tok->Decode(ids, true), c.Find("decoded")->as_string());+ for (const Value& s : c.Find("slices")->items()) {+ slices++;+ size_t start = size_t(s.Find("start")->as_int());+ size_t end = std::min(start + size_t(s.Find("size")->as_int()), ids.size());+ std::vector<int32_t> part(ids.begin() + long(start), ids.begin() + long(end));+ EXPECT_EQ(tok->Decode(part, true), s.Find("decoded")->as_string()) << "slice " << start;+ }+ if (c.Find("slices")->size() == 0) EXPECT_TRUE(text.empty() && ids.empty());+ const Value& w = *c.Find("with_specials");+ auto with = Ints(*w.Find("ids"));+ EXPECT_EQ(tok->Decode(with, true), w.Find("skipping_specials")->as_string());+ EXPECT_EQ(tok->Decode(with, false), w.Find("kept")->as_string());+ }+ EXPECT_GE(slices, 150u) << "the slice cases are the point of this family";+}++TEST_P(Shipped, EncodeFuzzMatchesTheCrate) {+ std::string name = GetParam();+ LAYA_GOLDEN_LIST(side, "go_extra.json", "encode_fuzz", name.c_str());+ LAYA_CHECKPOINT(tok, name);+ const Value& texts = *side->Find("texts");+ const Value& ids = *side->Find("ids");+ ASSERT_GE(texts.size(), 500u);+ int bad = 0;+ for (size_t i = 0; i < texts.size() && bad < 5; i++) {+ auto got = tok->Encode(texts.value(i).as_string());+ if (got != Ints(ids.value(i))) {+ ADD_FAILURE() << "Encode(" << Hex(texts.value(i).as_string()) << ") = " << Ids(got) << ", want " << Ids(Ints(ids.value(i)));+ bad++;+ }+ }+}++TEST_P(Shipped, DecodeFuzzMatchesTheCrate) {+ std::string name = GetParam();+ LAYA_GOLDEN_LIST(cases, "go_extra.json", "decode_fuzz", name.c_str(), "cases");+ LAYA_CHECKPOINT(tok, name);+ int bad = 0;+ for (const Value& c : cases->items()) {+ auto ids = Ints(*c.Find("ids"));+ for (bool skip : {true, false}) {+ if (tok->Decode(ids, skip) != c.Find(skip ? "skipping" : "keeping")->as_string() && bad++ < 5) {+ ADD_FAILURE() << "Decode(" << Ids(ids) << ", skip=" << skip << ")";+ }+ }+ }+}++// A Python str is always valid, so invalid bytes reach the tokenizer as bytes.decode('utf-8','replace').+TEST_P(Shipped, InvalidUtf8EncodesLikePython) {+ std::string name = GetParam();+ LAYA_GOLDEN_LIST(fam, "go_extra.json", "invalid_utf8");+ LAYA_CHECKPOINT(tok, name);+ const Value& hexes = *fam->Find("hex");+ const Value& want = *At(fam, {"ids", name.c_str()});+ ASSERT_GE(want.size(), 600u);+ for (size_t i = 0; i < want.size(); i++) {+ const std::string& h = hexes.value(i).as_string();+ std::string raw;+ for (size_t k = 0; k + 1 < h.size(); k += 2) raw.push_back(char(std::stoi(h.substr(k, 2), nullptr, 16)));+ ASSERT_EQ(tok->Encode(raw), Ints(want.value(i))) << h;+ }+}++// A loaded Tokenizer is read-only, so any number of threads may share it.+TEST_P(Shipped, ConcurrentUse) {+ std::string name = GetParam();+ LAYA_GOLDEN_LIST(side, "go_extra.json", "encode_fuzz", name.c_str());+ LAYA_CHECKPOINT(tok, name);+ const Value& texts = *side->Find("texts");+ const Value& ids = *side->Find("ids");+ std::vector<std::string> errors(8);+ std::vector<std::thread> threads;+ for (int g = 0; g < 8; g++) {+ threads.emplace_back([&, g] {+ for (size_t i = size_t(g); i < texts.size(); i += size_t(1 + g % 3)) {+ auto got = tok->Encode(texts.value(i).as_string());+ if (got != Ints(ids.value(i))) {+ errors[size_t(g)] = "thread " + std::to_string(g) + " text " + std::to_string(i);+ return;+ }+ (void)tok->Decode(got, true);+ }+ });+ }+ for (auto& t : threads) t.join();+ for (const auto& e : errors) EXPECT_EQ(e, "");+}++TEST(TokenizerDecode, DecoderChainsAndEdges) {+ LAYA_GOLDEN_LIST(by, "decode_text.json", "by");+ EXPECT_EQ(At(by, {"english", "decoder", "type"})->as_string(), "ByteLevel");+ std::string types;+ for (const Value& s : At(by, {"multilingual", "decoder", "decoders"})->items()) types += s.Find("type")->as_string() + ",";+ EXPECT_EQ(types, "Replace,ByteFallback,Fuse,");+}++TEST(TokenizerDecode, SyntheticDecoderShapesAndMixedToken) {+ LAYA_GOLDEN_LIST(shapes, "decode_text.json", "synthetic_decoders");+ ASSERT_GE(shapes->size(), 5u);+ for (size_t i = 0; i < shapes->size(); i++) {+ const Value& shape = shapes->value(i);+ auto spec = pv::Dumps(*shape.Find("spec"));+ ASSERT_TRUE(spec.ok());+ Tokenizer tok = MustParse(spec.value());+ ASSERT_GT(shape.Find("cases")->size(), 0u);+ for (const Value& c : shape.Find("cases")->items()) {+ EXPECT_EQ(tok.Decode(Ints(*c.Find("ids")), false), c.Find("decoded")->as_string()) << shapes->key(i);+ }+ if (const Value* panics = shape.Find("crate_panics")) { // the crate panics on these; this port returns empty text+ for (const Value& p : panics->items()) EXPECT_EQ(tok.Decode(Ints(p), false), "") << shapes->key(i);+ }+ }+ const Value* mixed = At(laya_test::Load("decode_text.json", true), {"mixed_token"});+ ASSERT_NE(mixed, nullptr);+ EXPECT_EQ(mixed->Find("scope")->as_string(), "token");+ auto spec = pv::Dumps(*mixed->Find("spec"));+ ASSERT_TRUE(spec.ok());+ Tokenizer tok = MustParse(spec.value());+ for (const Value& c : mixed->Find("cases")->items()) {+ EXPECT_EQ(tok.Decode(Ints(*c.Find("ids")), false), c.Find("decoded")->as_string()) << c.Find("why")->as_string();+ }+}++TEST(TokenizerDecode, EdgesOnTheEnglishAndMultilingualCheckpoints) {+ LAYA_CHECKPOINT(en, "english");+ LAYA_CHECKPOINT(ml, "multilingual");+ EXPECT_EQ(en->Decode({}, true), "");+ EXPECT_EQ(en->Decode({1 << 30}, true), "");+ int32_t a = en->Encode("a")[0];+ EXPECT_EQ(en->Decode({1 << 30, a, -1}, true), "a");+ EXPECT_EQ(en->Decode({50275}, true), " ") << "the literal-whitespace added token survives decoding";+ for (const char* s : {" ", " "}) EXPECT_EQ(en->Decode(en->Encode(s), true), s);+ EXPECT_EQ(ml->Decode(ml->Encode("Hello world"), true), " Hello world");+ auto ids = ml->Encode("\xF3\xB0\x80\x80"); // U+F0000, four bytes of byte fallback+ ASSERT_GE(ids.size(), 3u);+ EXPECT_EQ(ml->Decode({ids[0], ids[1], ids[2]}, true), " \xEF\xBF\xBD\xEF\xBF\xBD");+ EXPECT_EQ(ml->Decode(ids, true), " \xF3\xB0\x80\x80");+ auto full = en->Encode("hello world, this is a longer sentence");+ EXPECT_TRUE(en->Encode("hello world", 0).empty());+ for (size_t n = 1; n <= full.size(); n++) {+ EXPECT_EQ(en->Encode("hello world, this is a longer sentence", int64_t(n)), std::vector<int32_t>(full.begin(), full.begin() + long(n)));+ }+ EXPECT_EQ(en->Encode("hello", 100), en->Encode("hello"));+}++// ---- loader behaviour on hand-built tokenizers ------------------------------------------------------+TEST(TokenizerLoader, SmallTokenizerRoundTrips) {+ Tokenizer tok = MustParse(ByteLevelSpec(",\"decoder\":{\"type\":\"ByteLevel\"}"));+ auto ids = tok.Encode("ab ab");+ EXPECT_EQ(tok.Decode(ids, true), "ab ab");+ EXPECT_EQ(ids[0], 256);+ EXPECT_EQ(tok.VocabSize(), 257u);+ EXPECT_EQ(tok.HighestId(), 256);+ EXPECT_EQ(tok.IdToToken(256), std::optional<std::string>("ab"));+ EXPECT_FALSE(tok.IdToToken(-1).has_value());+ EXPECT_EQ(tok.TokenToId("ab"), std::optional<int32_t>(256));+}++TEST(TokenizerLoader, Refuses) {+ std::string good = ByteLevelSpec("");+ struct Case {+ const char* name;+ std::string spec;+ bool unsupported;+ } cases[] = {+ {"a non-BPE model", Replace(good, "\"BPE\"", "\"WordPiece\""), true},+ {"an unknown normalizer", Replace(good, "\"normalizer\":null", "\"normalizer\":{\"type\":\"BertNormalizer\"}"), true},+ {"an unknown pre-tokenizer", Replace(good, "\"pre_tokenizer\":{\"type\":\"ByteLevel\"", "\"pre_tokenizer\":{\"type\":\"Whitespace\""), true},+ {"an unknown decoder", ByteLevelSpec(",\"decoder\":{\"type\":\"WordPiece\"}"), true},+ {"a merge naming a symbol the vocabulary lacks", Replace(good, "\"a b\"", "\"a zz\""), false},+ {"a merge whose result is missing", Replace(good, "\"a b\"", "\"a c\""), false},+ {"a merge that is not a pair", Replace(good, "\"a b\"", "[\"a\"]"), false},+ {"a space-joined merge with an empty half", Replace(good, "\"a b\"", "\"a \""), false},+ {"an added token with no id", ByteLevelSpec(",\"added_tokens\":[{\"content\":\"x\"}]"), false},+ {"two added tokens sharing an id", ByteLevelSpec(",\"added_tokens\":[{\"content\":\"x\",\"id\":300},{\"content\":\"y\",\"id\":300}]"), false},+ {"a vocabulary giving one id to two tokens", Replace(good, "\"ab\":256", "\"ab\":0"), false},+ {"no vocabulary", "{\"model\":{\"type\":\"BPE\",\"merges\":[]}}", false},+ {"not JSON", "{", false},+ {"a lossy non-byte-level tokenizer", "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0},\"merges\":[]}}", true},+ };+ for (const auto& c : cases) {+ auto t = Tokenizer::Parse(c.spec);+ ASSERT_FALSE(t.ok()) << c.name << " must be refused, not loaded";+ EXPECT_EQ(tk::IsUnsupported(t.error()), c.unsupported) << c.name << ": " << t.error();+ }+ auto partial = Tokenizer::Parse("{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0},\"merges\":[]},\"pre_tokenizer\":{\"type\":\"ByteLevel\"}}");+ ASSERT_FALSE(partial.ok());+ EXPECT_NE(partial.error().find("absent from its vocabulary"), std::string::npos) << partial.error();+}++TEST(TokenizerLoader, UnknownTokenAndFuseUnk) {+ std::string with_unk = "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"<unk>\":1},\"merges\":[],\"unk_token\":\"<unk>\"}}";+ struct Case {+ bool fuse;+ const char* text;+ std::vector<int32_t> want;+ } cases[] = {{false, "azz a", {0, 1, 1, 1, 0}}, {false, "zzz", {1, 1, 1}}, {false, "zz a zz", {1, 1, 1, 0, 1, 1, 1}},+ {true, "azz a", {0, 1, 0}}, {true, "zzz", {1}}, {true, "zz a zz", {1, 0, 1}}};+ for (const auto& c : cases) {+ Tokenizer tok = MustParse(c.fuse ? Replace(with_unk, "\"unk_token\"", "\"fuse_unk\":true,\"unk_token\"") : with_unk);+ EXPECT_EQ(tok.Encode(c.text), c.want) << "fuse=" << c.fuse << " " << c.text;+ }+}++TEST(TokenizerLoader, IdsFarBeyondTheVocabularyAreRefused) {+ for (const char* id : {"2147483647", "1000000000", "70000"}) {+ auto t = Tokenizer::Parse(std::string("{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"<unk>\":") + id + "},\"merges\":[],\"unk_token\":\"<unk>\"}}");+ ASSERT_FALSE(t.ok()) << id;+ EXPECT_NE(t.error().find("sparse"), std::string::npos) << t.error();+ }+ EXPECT_TRUE(Tokenizer::Parse("{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0,\"<unk>\":9000},\"merges\":[],\"unk_token\":\"<unk>\"}}").ok());+}++TEST(TokenizerAdded, LongestLstripRstripSingleWordNormalized) {+ Tokenizer tok = MustParse(ByteLevelSpec(+ ",\"added_tokens\":[{\"content\":\"<S>\",\"id\":300,\"special\":true,\"normalized\":false},"+ "{\"content\":\"<SS>\",\"id\":301,\"special\":true,\"normalized\":false},"+ "{\"content\":\"[M]\",\"id\":302,\"special\":true,\"lstrip\":true,\"rstrip\":true},"+ "{\"content\":\"word\",\"id\":303,\"single_word\":true},"+ "{\"content\":\"\xC3\xA9" "clair\",\"id\":304,\"normalized\":true},"+ "{\"content\":\"\xC2\xA7\",\"id\":305},{\"content\":\"!\",\"id\":306}],\"decoder\":{\"type\":\"ByteLevel\"}"));+ using V = std::vector<int32_t>;+ EXPECT_EQ(tok.Encode("<SS><S>"), (V{301, 300})) << "the longest token wins at a position";+ EXPECT_EQ(tok.Encode("a [M] b"), (V{97, 302, 98})) << "lstrip and rstrip absorb the white space";+ EXPECT_EQ(tok.Encode("word"), (V{303}));+ auto inside = tok.Encode("swords word.");+ ASSERT_GE(inside.size(), 3u);+ EXPECT_EQ(inside[inside.size() - 2], 303) << "a single_word token inside a word is not matched but a free one is";+ EXPECT_EQ(tok.Encode("\xC3\xA9" "clair"), (V{304}));+ EXPECT_EQ(tok.Encode("a\xC2\xA7" "b!"), (V{97, 305, 98, 306}));+ EXPECT_EQ(tok.Encode("<S"), (V{60, 83})) << "a prefix of a token is not the token";+ EXPECT_EQ(tok.Decode({300, 97, 301}, true), "a");+ EXPECT_EQ(tok.Decode({300, 97}, false), "<S>a");+ EXPECT_EQ(tok.SpecialId("<S>"), std::optional<int32_t>(300));+ EXPECT_FALSE(tok.SpecialId("word").has_value());+ EXPECT_EQ(tok.SpecialTokens().size(), 3u);+}++TEST(TokenizerLoader, RolesFromTokenizerConfigAndDirectoryLayout) {+ std::string base = ::testing::TempDir() + "/laya_tok_roles";+ auto write = [](const std::string& path, const std::string& content) {+ std::ofstream(path, std::ios::binary) << content;+ };+ std::string cmd = "mkdir -p '" + base + "/m/tokenizer' '" + base + "/plain' '" + base + "/empty'";+ ASSERT_EQ(std::system(cmd.c_str()), 0);+ std::string spec = ByteLevelSpec(",\"added_tokens\":[{\"content\":\"<c>\",\"id\":300,\"special\":true},{\"content\":\"<m>\",\"id\":301,\"special\":true}]");+ write(base + "/plain/tokenizer.json", spec);+ write(base + "/plain/tokenizer_config.json", "{\"cls_token\":\"<c>\",\"mask_token\":{\"content\":\"<m>\"},\"pad_token\":null}");+ auto t = Tokenizer::LoadDir(base + "/plain");+ ASSERT_TRUE(t.ok()) << t.error();+ EXPECT_EQ(t.value().Cls(), std::optional<int32_t>(300));+ EXPECT_EQ(t.value().Mask(), std::optional<int32_t>(301));+ EXPECT_EQ(t.value().MaskToken(), "<m>");+ EXPECT_FALSE(t.value().Pad().has_value()) << "a null role is absent";+ EXPECT_FALSE(t.value().Sep().has_value()) << "an unnamed role is absent";+ write(base + "/plain/tokenizer_config.json", "{\"cls_token\":\"<nope>\"}");+ EXPECT_FALSE(Tokenizer::LoadDir(base + "/plain").ok()) << "a role naming a token the vocabulary lacks";+ write(base + "/plain/tokenizer_config.json", "{\"cls_token\":5}");+ EXPECT_FALSE(Tokenizer::LoadDir(base + "/plain").ok()) << "a role that is not a string or an object";+ write(base + "/m/tokenizer/tokenizer.json", ByteLevelSpec(""));+ EXPECT_TRUE(Tokenizer::LoadDir(base + "/m").ok()) << "<dir>/tokenizer/ is preferred";+ EXPECT_FALSE(Tokenizer::LoadDir(base + "/empty").ok()) << "a directory with no tokenizer";+}++TEST(TokenizerRefusals, NormalizersPreTokenizersAndRegexDialects) {+ for (const char* spec : {"{\"type\":\"BertNormalizer\"}", "{}", "{\"type\":\"Replace\",\"pattern\":{\"String\":\"a\"}}",+ "{\"type\":\"Replace\",\"pattern\":{\"String\":\"\"},\"content\":\"x\"}",+ "{\"type\":\"Replace\",\"pattern\":{},\"content\":\"x\"}",+ "{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"(\"},\"content\":\"x\"}", "{\"type\":\"Sequence\"}",+ "{\"type\":\"Sequence\",\"normalizers\":[1]}"}) {+ EXPECT_FALSE(tk::RunNormalizerSpec(spec, "a").ok()) << spec;+ }+ // The reference reads \s, \d, \w, \b, \p as Unicode; an ASCII engine would differ, so they are refused.+ for (const char* expr : {"\\\\s+", "a\\\\bb", "[\\\\d]", "\\\\w", "\\\\p{L}"}) {+ auto r = tk::RunNormalizerSpec(std::string("{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"") + expr + "\"},\"content\":\"x\"}", "a");+ ASSERT_FALSE(r.ok()) << expr;+ EXPECT_TRUE(tk::IsUnsupported(r.error())) << expr << ": " << r.error();+ }+ auto esc = tk::RunNormalizerSpec("{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"\\\\\\\\\\\\\\\\s\"},\"content\":\"x\"}", "a\\\\s");+ ASSERT_TRUE(esc.ok()) << esc.error();+ EXPECT_EQ(esc.value(), "ax") << "an escaped backslash before s is a literal, not a class";+ auto empty = tk::RunNormalizerSpec("{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"a*\"},\"content\":\"x\"}", "b");+ ASSERT_FALSE(empty.ok());+ EXPECT_TRUE(tk::IsUnsupported(empty.error())) << "a pattern that can match nothing";+ for (const char* spec : {"{\"type\":\"Whitespace\"}", "{}", "{\"type\":\"Metaspace\",\"replacement\":\"ab\"}",+ "{\"type\":\"Metaspace\",\"prepend_scheme\":\"sometimes\"}",+ "{\"type\":\"Sequence\",\"pretokenizers\":[1]}"}) {+ EXPECT_FALSE(tk::RunPreTokenizerSpec(spec, "a").ok()) << spec;+ }+ EXPECT_TRUE(tk::IsUnsupported(tk::RunPreTokenizerSpec("{\"type\":\"Whitespace\"}", "a").error()));+}++TEST(TokenizerRegex, SubsetBehaviour) {+ auto run = [](const std::string& pattern, const std::string& in, const std::string& to = "_") {+ auto r = tk::RunNormalizerSpec("{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"" + pattern + "\"},\"content\":\"" + to + "\"}", in);+ EXPECT_TRUE(r.ok()) << pattern << ": " << r.error();+ return r.ok() ? r.value() : std::string("<error>");+ };+ EXPECT_EQ(run("a|b", "cabd"), "c__d");+ EXPECT_EQ(run("a+?", "aaa"), "___") << "lazy repeat";+ EXPECT_EQ(run("a+", "baaac"), "b_c") << "greedy repeat";+ EXPECT_EQ(run("(ab)+c", "ababc abc"), "_ _");+ EXPECT_EQ(run("[^a-c]", "abxcy"), "ab_c_");+ EXPECT_EQ(run("^a", "aa"), "_a");+ EXPECT_EQ(run("a$", "aa"), "a_");+ EXPECT_EQ(run("a.c", "abc a\\nc"), "_ a\\nc");+ EXPECT_EQ(run("x{2,3}", "xxxxxxx"), "__x");+ EXPECT_EQ(run("(?i)HeLLo", "Hello HELLO hello"), "_ _ _");+ EXPECT_EQ(run("é+", "ééa"), "_a");+ EXPECT_EQ(run("(a|ab)(c|bcd)", "abcd"), "_") << "leftmost-first: a then bcd";+ // The failed-state memo keeps nested repeats linear.+ EXPECT_EQ(run("(a*)*b", std::string(5000, 'a')), std::string(5000, 'a'));+}++TEST(TokenizerPre, SequenceAndCornerCases) {+ auto seq = tk::RunPreTokenizerSpec(+ "{\"type\":\"Sequence\",\"pretokenizers\":[{\"type\":\"Metaspace\",\"prepend_scheme\":\"never\"},{\"type\":\"ByteLevel\",\"add_prefix_space\":false}]}", "ab cd");+ ASSERT_TRUE(seq.ok());+ EXPECT_EQ(Join(seq.value()), Join({"ab", "\xC3\xA2\xC4\xB8\xC4\xA3", "cd"})) << "the crate gives ['ab', 'âĸģ', 'cd']";+ const char* spec = "{\"type\":\"ByteLevel\",\"add_prefix_space\":false,\"use_regex\":true}";+ struct Case {+ const char* in;+ std::vector<std::string> want;+ } cases[] = {+ {"a b\t\tc\n\nd\r\ne f", {"a", "\xC4\xA0", "\xC4\xA0" "b", "\xC4\x89", "\xC4\x89", "c", "\xC4\x8A", "\xC4\x8A", "d", "\xC4\x8D", "\xC4\x8A", "e", "\xC4\xA0\xC4\xA0", "\xC4\xA0" "f"}},+ {"abc123", {"abc", "123"}}, {"123abc", {"123", "abc"}}, {"a 1 b", {"a", "\xC4\xA0" "1", "\xC4\xA0" "b"}},+ {"x's value", {"x", "'s", "\xC4\xA0" "value"}}, {"", {}}};+ for (const auto& c : cases) {+ auto got = tk::RunPreTokenizerSpec(spec, c.in);+ ASSERT_TRUE(got.ok());+ EXPECT_EQ(Join(got.value()), Join(c.want)) << c.in;+ }+}diff --git a/laya-go/scripts/gen_tables.py b/laya-go/scripts/gen_tables.pyindex 9aa3a986..8a4ff32b 100644--- a/laya-go/scripts/gen_tables.py+++ b/laya-go/scripts/gen_tables.py@@ -37,6 +37,7 @@ OUT_LANG = os.path.join(GO, "lang", "tables_gen.go")OUT_TOKENIZER = os.path.join(GO, "tokenizer", "tables_gen.go")OUT_CPP_UNICODE = os.path.join(REPO, "laya-cpp", "src", "unicodedata_tables.inc")OUT_CPP_LANG = os.path.join(REPO, "laya-cpp", "src", "lang_tables.inc")+OUT_CPP_TOKENIZER = os.path.join(REPO, "laya-cpp", "src", "tokenizer_tables.inc")def wrap(values, indent="\t", width=100):@@ -74,6 +75,14 @@ def table(name, rows, doc):doc, len(rows), sum(b - a + 1 for a, b in rows), name, wrap(flat))+def tok_array(cpp, go_name, cpp_name, values, doc, ctype="uint32_t"):+ """One generated array of the tokenizer tables, as Go (`var goName = [...]rune{`) or C++."""+ if cpp:+ return "// %s\ninline constexpr %s %s[] = {\n%s\n};\n\n" % (doc, ctype, cpp_name, wrap(values, indent=" "))+ gotype = "uint64" if ctype == "uint64_t" else "uint32" if ctype == "uint32_t" and go_name.endswith("Offsets") else "rune"+ return "// %s\nvar %s = [...]%s{\n%s\n}\n\n" % (doc, go_name, gotype, wrap(values))++def ascii_masks(name, predicate):low = sum(1 << cp for cp in range(0, 64) if predicate(cp))high = sum(1 << (cp - 64) for cp in range(64, 128) if predicate(cp))@@ -327,13 +336,26 @@ def render_lang_cpp():# ---------------------------------------------------------------------------------------------# The tokenizers crate: pre-tokenizer classes, single_word flank, lowercase, normalization.# ----------------------------------------------------------------------------------------------def render_tokenizer():+def render_tokenizer(cpp=False):import tokenizersfrom tokenizers import AddedToken, Tokenizer, models, normalizers, pre_tokenizers- out = [header("tokenizers %s" % tokenizers.__version__), "package tokenizer\n\n"]- out.append("// Reference names the implementation the tables below were probed from.\n"- "const Reference = \"tokenizers %s\"\n\n" % tokenizers.__version__)+ if cpp:+ out = ["// Code generated by laya-go/scripts/gen_tables.py --lang cpp from tokenizers %s; DO NOT EDIT.\n//\n"+ "// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp\n\n"+ "inline constexpr const char* kTokenizerReference = \"tokenizers %s\";\n\n"+ % (tokenizers.__version__, tokenizers.__version__)]+ else:+ out = [header("tokenizers %s" % tokenizers.__version__), "package tokenizer\n\n"]+ out.append("// Reference names the implementation the tables below were probed from.\n"+ "const Reference = \"tokenizers %s\"\n\n" % tokenizers.__version__)++ def rng(go_name, cpp_name, rows, doc):+ if not cpp:+ return table(go_name, rows, doc)+ flat = [v for row in rows for v in row]+ return tok_array(True, go_name, cpp_name, flat, "%s\n// %d ranges covering %d code points."+ % (doc.replace("\n//", "\n//"), len(rows), sum(b - a + 1 for a, b in rows)))# Letter / number: "a<cp>" stays ONE piece only if ` ?\p{L}+` consumed the code point;# "1<cp>" only if ` ?\p{N}+` did. Anything else splits.@@ -347,9 +369,9 @@ def render_tokenizer():return Falsereturn test- out.append(table("letter", ranges(one_piece("a")),+ out.append(rng("letter", "k_letter", ranges(one_piece("a")),"letter is the pre-tokenizer pattern's \\p{L}, probed from the reference."))- out.append(table("number", ranges(one_piece("1")),+ out.append(rng("number", "k_number", ranges(one_piece("1")),"number is the pre-tokenizer pattern's \\p{N}, probed from the reference."))# single_word: an added token is refused when the adjacent character is a word character.@@ -367,7 +389,7 @@ def render_tokenizer():raise SystemExit("gen_tables: single_word treats U+%04X differently on each side" % cp)return left- out.append(table("wordChar", ranges(flanks),+ out.append(rng("wordChar", "k_word_char", ranges(flanks),"wordChar is what the reference counts as a word character beside a\n""// single_word added token."))@@ -384,16 +406,25 @@ def render_tokenizer():simple.append((cp, ord(mapped)))else:multi.append((cp, [ord(c) for c in mapped]))- out.append("var lowerFrom = [...]rune{\n%s\n}\n\n" % wrap([c for c, _ in simple]))- out.append("var lowerTo = [...]rune{\n%s\n}\n\n" % wrap([t for _, t in simple]))- out.append("var lowerMulti = map[rune][]rune{\n%s\n}\n\n"- % "\n".join("\t%#x: {%s}," % (cp, ", ".join("%#x" % c for c in m)) for cp, m in multi))-- out.append(render_normalization(normalizers))+ if cpp:+ out.append(tok_array(True, "", "k_lower_from", [c for c, _ in simple], "lowercase, one code point to one"))+ out.append(tok_array(True, "", "k_lower_to", [t for _, t in simple], "the lowercase of k_lower_from"))+ assert all(len(m) <= 3 for _, m in multi)+ out.append("struct TokLowerMulti { uint32_t from; uint32_t count; uint32_t to[3]; };\n"+ "inline constexpr TokLowerMulti k_lower_multi[] = {\n%s\n};\n\n"+ % "\n".join(" {%#x, %d, {%s}}," % (cp, len(m), ", ".join("%#x" % c for c in m + [0] * (3 - len(m))))+ for cp, m in multi))+ else:+ out.append("var lowerFrom = [...]rune{\n%s\n}\n\n" % wrap([c for c, _ in simple]))+ out.append("var lowerTo = [...]rune{\n%s\n}\n\n" % wrap([t for _, t in simple]))+ out.append("var lowerMulti = map[rune][]rune{\n%s\n}\n\n"+ % "\n".join("\t%#x: {%s}," % (cp, ", ".join("%#x" % c for c in m)) for cp, m in multi))++ out.append(render_normalization(normalizers, cpp))return "".join(out)-def render_normalization(normalizers):+def render_normalization(normalizers, cpp=False):nfc, nfd = normalizers.NFC(), normalizers.NFD()nfkd = normalizers.NFKD()canon, compat = {}, {}@@ -484,6 +515,10 @@ def render_normalization(normalizers):for k in ks:flat.extend(mapping[k])offsets.append(len(flat))+ if cpp:+ return (tok_array(True, "", "k_%s_keys" % name, ks, doc)+ + tok_array(True, "", "k_%s_offsets" % name, offsets, "offsets into k_%s_pool" % name)+ + tok_array(True, "", "k_%s_pool" % name, flat, "the decompositions of k_%s_keys" % name))return ("// %s\nvar %sKeys = [...]rune{\n%s\n}\n\nvar %sOffsets = [...]uint32{\n%s\n}\n\n""var %sPool = [...]rune{\n%s\n}\n\n"% (doc, name, wrap(ks), name, wrap(offsets), name, wrap(flat)))@@ -493,18 +528,24 @@ def render_normalization(normalizers):"// point that has one, except Hangul syllables, which are algorithmic."))text.append(pool(compat, "compat", "compat* hold the full compatibility decomposition (NFKD) of every code\n""// point where it differs from the canonical one."))- text.append("// cccRanges are (lo, hi, class) triples of the canonical combining class, sorted.\n"- "var cccRanges = [...]rune{\n%s\n}\n\n" % wrap([v for r in rows for v in r]))- text.append("// composeKeys are (first<<32 | second) of each primary composition pair, sorted;\n"- "// composeValues is the composite.\nvar composeKeys = [...]uint64{\n%s\n}\n\n"- % wrap(["%#x" % ((a << 32) | b) for a, b in keys]))+ ccc_doc = "cccRanges are (lo, hi, class) triples of the canonical combining class, sorted."+ comp_doc = "composeKeys are (first<<32 | second) of each primary composition pair, sorted;\n// composeValues is the composite."+ if cpp:+ text.append(tok_array(True, "", "k_ccc_ranges", [v for r in rows for v in r], ccc_doc))+ text.append(tok_array(True, "", "k_compose_keys", ["%#xULL" % ((a << 32) | b) for a, b in keys], comp_doc, "uint64_t"))+ text.append(tok_array(True, "", "k_compose_values", [pairs[k] for k in keys], "the composite of k_compose_keys"))+ return "".join(text)+ text.append("// %s\nvar cccRanges = [...]rune{\n%s\n}\n\n" % (ccc_doc, wrap([v for r in rows for v in r])))+ text.append("// %s\nvar composeKeys = [...]uint64{\n%s\n}\n\n"+ % (comp_doc, wrap(["%#x" % ((a << 32) | b) for a, b in keys])))text.append("var composeValues = [...]rune{\n%s\n}\n" % wrap([pairs[k] for k in keys]))return "".join(text)TARGETS = ((OUT_UNICODE, render_unicode), (OUT_LANG, render_lang), (OUT_TOKENIZER, render_tokenizer))-# laya-cpp: the tokenizer tables join in C2.-CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)), (OUT_CPP_LANG, render_lang_cpp))+# laya-cpp generated tables.+CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)), (OUT_CPP_LANG, render_lang_cpp),+ (OUT_CPP_TOKENIZER, lambda: render_tokenizer(cpp=True)))def main(argv=None):