Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(cpp): sha256, pyjson/pyvalue and CPython unicode tables

Adeshina Lasisi1fa70279e722
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 5b637757..a8dba4d0 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -37,7 +37,10 @@ concurrency:
env:
# The test floor: the number of ctest tests laya-cpp registers (the GoogleTest cases plus
# compile_flags). Raise it with every test added.
- MIN_TESTS: "2"
+ MIN_TESTS: "95"
+ GOLDENS: ${{ github.workspace }}/.cpp-goldens
+ LAYA_GOLDENS: ${{ github.workspace }}/.cpp-goldens
+ LAYA_REQUIRE_GOLDENS: "1" # a missing golden is a failure, never a skip
jobs:
build:
@@ -56,10 +59,37 @@ jobs:
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
+ cache: pip
+ cache-dependency-path: laya-dotnet/tools/requirements-regen.txt
- name: Checker tests
run: python laya-cpp/scripts/test_check_test_xml.py
+ # Goldens are recorded from the Python reference at this commit, never read from a committed copy.
+ - name: Install the Python toolchain the goldens are recorded with
+ run: |
+ python -m pip install --upgrade pip
+ pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu
+ pip install -r laya-dotnet/tools/requirements-regen.txt
+ - name: Fetch the tokenizers the goldens are recorded against
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
+ run: |
+ for c in english multilingual; do
+ python laya-java/scripts/prepare_checkpoint.py --checkpoint "$c" --no-graph
+ done
+ - name: Regenerate the goldens from Python
+ env:
+ LAYA_FIXTURE_CHECKPOINTS: ${{ github.workspace }}/laya-java/.work/checkpoints
+ PYTHONPATH: ${{ github.workspace }}
+ run: |
+ python laya-java/scripts/gen_fixtures.py --strict --out "$GOLDENS" --only lang_tables.json --only lang_detect.json --only python_json.json
+ python laya-go/scripts/gen_go_fixtures.py --out "$GOLDENS"
+ - name: Compiled C++ tables match CPython
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ run: python laya-go/scripts/gen_tables.py --lang cpp --check
+
- name: Configure
working-directory: laya-cpp
run: cmake --preset "${{ matrix.preset }}" -DCMAKE_CXX_COMPILER="${{ matrix.compiler }}"
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txt
index f0e871be..542a7a18 100644
--- a/laya-cpp/CMakeLists.txt
+++ b/laya-cpp/CMakeLists.txt
@@ -25,7 +25,7 @@ function(laya_target_flags target)
target_compile_options(${target} PRIVATE ${LAYA_FP_FLAGS} ${LAYA_WARN_FLAGS})
endfunction()
-add_library(laya src/version.cpp)
+add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp)
target_include_directories(laya PUBLIC include)
laya_target_flags(laya)
@@ -44,12 +44,13 @@ if(LAYA_BUILD_TESTS)
FetchContent_MakeAvailable(googletest)
enable_testing()
- add_executable(laya_tests tests/version_test.cpp)
+ add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp)
target_link_libraries(laya_tests PRIVATE laya GTest::gtest_main)
laya_target_flags(laya_tests)
include(GoogleTest)
- gtest_discover_tests(laya_tests)
+ # PRE_TEST: the parametrized rows are counted from the goldens, which exist at test time.
+ gtest_discover_tests(laya_tests DISCOVERY_MODE PRE_TEST)
# compile_commands.json is the record of what the compiler was really given, so this fails if
# the floating point flags are dropped from either target.
diff --git a/laya-cpp/include/laya/pyjson.hpp b/laya-cpp/include/laya/pyjson.hpp
new file mode 100644
index 00000000..b46940b3
--- /dev/null
+++ b/laya-cpp/include/laya/pyjson.hpp
@@ -0,0 +1,25 @@
+#pragma once
+// Text the way CPython writes it: repr(float), repr(str), '%.0f' and round(x, 4). These feed the
+// tokenizer, so one differing character is a different answer; never use %g, ostream or locale
+// formatting for them.
+#include <string>
+#include <string_view>
+
+namespace laya {
+namespace pyjson {
+
+constexpr int kMaxDepth = 512;
+
+// repr(float) as json.dumps writes it: shortest round-trip digits, fixed notation for decimal
+// exponents in [-4, 16), else scientific with a signed two-digit exponent. NaN, Infinity,
+// -Infinity as json.dumps spells them.
+std::string ReprFloat(double v);
+// repr(str): single-quoted unless the text holds ' and no ", everything not isprintable escaped.
+std::string Repr(std::string_view s);
+// '%.0f' % (100 * fraction): halves to even on the exact binary value.
+std::string Percent0(double fraction);
+// round(x, 4): correctly rounded, ties to even on the exact binary value. NaN and inf pass through.
+double Round4(double v);
+
+} // namespace pyjson
+} // namespace laya
diff --git a/laya-cpp/include/laya/pyvalue.hpp b/laya-cpp/include/laya/pyvalue.hpp
new file mode 100644
index 00000000..adf49c8e
--- /dev/null
+++ b/laya-cpp/include/laya/pyvalue.hpp
@@ -0,0 +1,71 @@
+#pragma once
+// The values laya's Python code handles as dicts, lists, strs, ints and floats, read and written
+// as json.loads / json.dumps(ensure_ascii=False) do: object order and number spelling kept, NaN
+// and Infinity accepted, a repeated key keeps its first position and takes its last value.
+#include <cstdint>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "laya/result.hpp"
+
+namespace laya {
+namespace pyvalue {
+
+constexpr int kMaxDepth = 512; // nesting bound for Parse and Dumps
+constexpr int kMaxIntDigits = 4300; // sys.int_info.default_max_str_digits
+
+class Value {
+ public:
+ enum class Kind { Null, Bool, Int, BigInt, Float, String, Array, Object };
+
+ Value() = default;
+ static Value Null() { return Value(); }
+ static Value Bool(bool b);
+ static Value Int(int64_t i);
+ static Value BigInt(std::string decimal); // an int literal beyond int64: sign and digits
+ static Value Float(double d);
+ static Value String(std::string s);
+ static Value Array(std::vector<Value> items = {});
+ static Value Object(); // empty; use Set to add members
+
+ Kind kind() const { return kind_; }
+ bool is_null() const { return kind_ == Kind::Null; }
+ bool as_bool() const { return i_ != 0; }
+ int64_t as_int() const { return i_; }
+ const std::string& big_int() const { return s_; }
+ double as_float() const { return d_; }
+ const std::string& as_string() const { return s_; }
+ const std::vector<Value>& items() const { return vals_; } // Array elements or Object values
+
+ // Object members in insertion order.
+ size_t size() const { return vals_.size(); }
+ const std::string& key(size_t i) const { return keys_[i]; }
+ const Value& value(size_t i) const { return vals_[i]; }
+ const Value* Find(std::string_view key) const; // nullptr when absent
+ // Python dict assignment: a new key goes last, an existing key keeps its place.
+ void Set(std::string key, Value v);
+ void Push(Value v) { vals_.push_back(std::move(v)); }
+
+ private:
+ Kind kind_ = Kind::Null;
+ int64_t i_ = 0;
+ double d_ = 0;
+ std::string s_;
+ std::vector<std::string> keys_;
+ std::vector<Value> vals_;
+};
+
+// json.loads of one document of UTF-8 text. Refused: invalid UTF-8, a lone surrogate escape, a raw
+// control character in a string, an int literal over 4300 digits, nesting beyond kMaxDepth.
+Result<Value> Parse(std::string_view text);
+// json.dumps(v, ensure_ascii=False) with the default ", " and ": " separators. Fails on a string
+// that is not valid UTF-8 and on nesting beyond kMaxDepth.
+Result<std::string> Dumps(const Value& v);
+// type(v).__name__, repr(v) and str(v) of a decoded value.
+std::string TypeName(const Value& v);
+std::string Repr(const Value& v);
+std::string Str(const Value& v);
+
+} // namespace pyvalue
+} // namespace laya
diff --git a/laya-cpp/include/laya/result.hpp b/laya-cpp/include/laya/result.hpp
new file mode 100644
index 00000000..beb3539c
--- /dev/null
+++ b/laya-cpp/include/laya/result.hpp
@@ -0,0 +1,47 @@
+#pragma once
+// Error handling for the whole library: no exception crosses the public API. A function that can
+// fail returns Result<T>; test ok() before value(). value() on a failed Result is a programming
+// error and aborts, it never throws.
+#include <cstdlib>
+#include <string>
+#include <utility>
+
+namespace laya {
+
+template <class T>
+class Result {
+ public:
+ static Result Ok(T value) {
+ Result r;
+ r.value_ = std::move(value);
+ r.ok_ = true;
+ return r;
+ }
+ static Result Fail(std::string message) {
+ Result r;
+ r.error_ = std::move(message);
+ return r;
+ }
+ bool ok() const { return ok_; }
+ const std::string& error() const { return error_; }
+ const T& value() const& {
+ if (!ok_) std::abort();
+ return value_;
+ }
+ T& value() & {
+ if (!ok_) std::abort();
+ return value_;
+ }
+ T&& value() && {
+ if (!ok_) std::abort();
+ return std::move(value_);
+ }
+
+ private:
+ Result() = default;
+ T value_{};
+ std::string error_;
+ bool ok_ = false;
+};
+
+} // namespace laya
diff --git a/laya-cpp/include/laya/sha256.hpp b/laya-cpp/include/laya/sha256.hpp
new file mode 100644
index 00000000..46ab13da
--- /dev/null
+++ b/laya-cpp/include/laya/sha256.hpp
@@ -0,0 +1,32 @@
+#pragma once
+// SHA-256 (FIPS 180-4), self-contained. Used for artifact digests, not for secrets.
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <string_view>
+
+namespace laya {
+
+class Sha256 {
+ public:
+ Sha256();
+ void Update(const void* data, size_t size);
+ void Update(std::string_view s) { Update(s.data(), s.size()); }
+ // Finishes the hash; the object must not be updated afterwards.
+ std::array<uint8_t, 32> Finish();
+
+ private:
+ void Block(const uint8_t* p);
+ uint32_t h_[8];
+ uint8_t buf_[64];
+ size_t used_ = 0;
+ uint64_t total_ = 0;
+};
+
+std::array<uint8_t, 32> Sha256Of(std::string_view data);
+// Lowercase hex, as hashlib.sha256(...).hexdigest().
+std::string Sha256Hex(std::string_view data);
+std::string ToHex(const std::array<uint8_t, 32>& digest);
+
+} // namespace laya
diff --git a/laya-cpp/include/laya/unicodedata.hpp b/laya-cpp/include/laya/unicodedata.hpp
new file mode 100644
index 00000000..0ef0496f
--- /dev/null
+++ b/laya-cpp/include/laya/unicodedata.hpp
@@ -0,0 +1,53 @@
+#pragma once
+// The character questions laya's Python reference answers with str.isalpha(), str.lower(), re's
+// \w and friends. Every predicate is a table recorded from CPython (kVersion), generated by
+// laya-go/scripts/gen_tables.py --lang cpp, so every compiler and OS gives one answer. Strings are
+// UTF-8 bytes; bytes that are not valid UTF-8 decode as Go's range loop does, one U+FFFD per byte.
+#include <cstdint>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace laya {
+namespace unicodedata {
+
+extern const char* const kVersion; // the Unicode release of the tables, e.g. "15.0.0"
+constexpr char32_t kReplacement = 0xFFFD;
+
+bool IsAlpha(char32_t r); // str.isalpha()
+bool IsWord(char32_t r); // [^\W\d_]
+bool IsCombining(char32_t r); // non-zero canonical combining class
+bool IsDigit(char32_t r); // \d (Nd)
+bool IsUpper(char32_t r);
+bool IsLower(char32_t r);
+bool IsSpace(char32_t r);
+bool IsMark(char32_t r); // Mn, Mc, Me
+bool IsInitial(char32_t r); // Lu, Lt, Lo
+bool IsPrintable(char32_t r);
+bool IsPythonWord(char32_t r); // \w
+bool IsWordOrDigit(char32_t r); // [^\W_]
+bool IsUpperString(std::string_view s); // str.isupper()
+
+// UTF-8 helpers with Go's semantics: an invalid sequence is (kReplacement, width 1).
+struct Decoded {
+ char32_t rune;
+ size_t width;
+};
+Decoded DecodeRune(std::string_view s); // first rune; s must be non-empty
+Decoded DecodeLastRune(std::string_view s); // last rune; s must be non-empty
+bool ValidUtf8(std::string_view s);
+void AppendRune(std::string& out, char32_t r); // invalid scalar values append U+FFFD
+size_t RuneCount(std::string_view s);
+
+char32_t LowerRune(char32_t r);
+std::string Lower(std::string_view s); // str.lower(), final sigma included
+std::string Lossy(std::string_view bytes); // bytes.decode("utf-8", "replace")
+std::string ToValid(std::string_view s); // same as Lossy; no copy needed when valid is the caller's job
+std::string DecimalASCII(std::string_view s);
+std::string Strip(std::string_view s); // str.strip()
+std::string StripNumeric(std::string_view s);
+bool IsBlank(std::string_view s);
+std::vector<std::string> Fields(std::string_view s); // str.split()
+
+} // namespace unicodedata
+} // namespace laya
diff --git a/laya-cpp/src/pyjson.cpp b/laya-cpp/src/pyjson.cpp
new file mode 100644
index 00000000..1b26a369
--- /dev/null
+++ b/laya-cpp/src/pyjson.cpp
@@ -0,0 +1,113 @@
+#include "laya/pyjson.hpp"
+
+#include <charconv>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+
+#include "laya/unicodedata.hpp"
+
+namespace laya {
+namespace pyjson {
+
+namespace {
+std::string Fixed(const std::string& digits, int exp) {
+ if (exp < 0) return "0." + std::string(size_t(-exp - 1), '0') + digits;
+ size_t whole = size_t(exp) + 1;
+ if (digits.size() <= whole) return digits + std::string(whole - digits.size(), '0') + ".0";
+ return digits.substr(0, whole) + "." + digits.substr(whole);
+}
+
+std::string Hex(char32_t r, int width, char marker) {
+ char buf[16];
+ std::snprintf(buf, sizeof buf, "\\%c%0*x", marker, width, static_cast<unsigned>(r));
+ return buf;
+}
+} // namespace
+
+std::string ReprFloat(double v) {
+ if (std::isnan(v)) return "NaN";
+ if (std::isinf(v)) return v > 0 ? "Infinity" : "-Infinity";
+ std::string sign;
+ if (std::signbit(v)) {
+ sign = "-";
+ v = -v;
+ }
+ if (v == 0) return sign + "0.0";
+ char buf[64];
+ auto res = std::to_chars(buf, buf + sizeof buf, v, std::chars_format::scientific); // shortest d.ddde+XX
+ std::string text(buf, res.ptr);
+ size_t e = text.find('e');
+ std::string mantissa = text.substr(0, e);
+ int exp = std::atoi(text.c_str() + e + 1);
+ std::string digits;
+ for (char c : mantissa) {
+ if (c != '.') digits.push_back(c);
+ }
+ if (exp >= -4 && exp < 16) return sign + Fixed(digits, exp);
+ std::string out = sign;
+ out.push_back(digits[0]);
+ if (digits.size() > 1) {
+ out.push_back('.');
+ out += digits.substr(1);
+ }
+ out.push_back('e');
+ out.push_back(exp < 0 ? '-' : '+');
+ if (exp < 0) exp = -exp;
+ if (exp < 10) out.push_back('0');
+ out += std::to_string(exp);
+ return out;
+}
+
+std::string Repr(std::string_view s) {
+ bool has_single = s.find('\'') != std::string_view::npos;
+ bool has_double = s.find('"') != std::string_view::npos;
+ const char quote = (has_single && !has_double) ? '"' : '\'';
+ std::string out;
+ out.reserve(s.size() + 2);
+ out.push_back(quote);
+ for (size_t i = 0; i < s.size();) {
+ unicodedata::Decoded d = unicodedata::DecodeRune(s.substr(i));
+ i += d.width;
+ const char32_t r = d.rune;
+ if (r == char32_t(quote) || r == '\\') {
+ out.push_back('\\');
+ out.push_back(char(r));
+ } else if (r == '\n') {
+ out += "\\n";
+ } else if (r == '\r') {
+ out += "\\r";
+ } else if (r == '\t') {
+ out += "\\t";
+ } else if (unicodedata::IsPrintable(r)) {
+ unicodedata::AppendRune(out, r);
+ } else if (r < 0x100) {
+ out += Hex(r, 2, 'x');
+ } else if (r < 0x10000) {
+ out += Hex(r, 4, 'u');
+ } else {
+ out += Hex(r, 8, 'U');
+ }
+ }
+ out.push_back(quote);
+ return out;
+}
+
+std::string Percent0(double fraction) {
+ double scaled = 100 * fraction;
+ if (std::isnan(scaled)) return "nan";
+ if (std::isinf(scaled)) return scaled > 0 ? "inf" : "-inf";
+ char buf[400];
+ std::snprintf(buf, sizeof buf, "%.0f", scaled);
+ return buf;
+}
+
+double Round4(double v) {
+ if (std::isnan(v) || std::isinf(v)) return v;
+ char buf[400];
+ std::snprintf(buf, sizeof buf, "%.4f", v);
+ return std::strtod(buf, nullptr);
+}
+
+} // namespace pyjson
+} // namespace laya
diff --git a/laya-cpp/src/pyvalue.cpp b/laya-cpp/src/pyvalue.cpp
new file mode 100644
index 00000000..b568fb19
--- /dev/null
+++ b/laya-cpp/src/pyvalue.cpp
@@ -0,0 +1,465 @@
+#include "laya/pyvalue.hpp"
+
+#include <cerrno>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <unordered_map>
+
+#include "laya/pyjson.hpp"
+#include "laya/unicodedata.hpp"
+
+namespace laya {
+namespace pyvalue {
+
+Value Value::Bool(bool b) {
+ Value v;
+ v.kind_ = Kind::Bool;
+ v.i_ = b ? 1 : 0;
+ return v;
+}
+Value Value::Int(int64_t i) {
+ Value v;
+ v.kind_ = Kind::Int;
+ v.i_ = i;
+ return v;
+}
+Value Value::BigInt(std::string decimal) {
+ Value v;
+ v.kind_ = Kind::BigInt;
+ v.s_ = std::move(decimal);
+ return v;
+}
+Value Value::Float(double d) {
+ Value v;
+ v.kind_ = Kind::Float;
+ v.d_ = d;
+ return v;
+}
+Value Value::String(std::string s) {
+ Value v;
+ v.kind_ = Kind::String;
+ v.s_ = std::move(s);
+ return v;
+}
+Value Value::Array(std::vector<Value> items) {
+ Value v;
+ v.kind_ = Kind::Array;
+ v.vals_ = std::move(items);
+ return v;
+}
+Value Value::Object() {
+ Value v;
+ v.kind_ = Kind::Object;
+ return v;
+}
+
+const Value* Value::Find(std::string_view k) const {
+ for (size_t i = 0; i < keys_.size(); i++) {
+ if (keys_[i] == k) return &vals_[i];
+ }
+ return nullptr;
+}
+
+void Value::Set(std::string k, Value v) {
+ for (size_t i = 0; i < keys_.size(); i++) {
+ if (keys_[i] == k) {
+ vals_[i] = std::move(v);
+ return;
+ }
+ }
+ keys_.push_back(std::move(k));
+ vals_.push_back(std::move(v));
+}
+
+namespace {
+
+class Parser {
+ public:
+ explicit Parser(std::string_view s) : s_(s) {}
+
+ Result<Value> Document() {
+ if (!unicodedata::ValidUtf8(s_)) return Result<Value>::Fail("JSON text is not valid UTF-8");
+ Space();
+ Result<Value> v = Val(0);
+ if (!v.ok()) return v;
+ Space();
+ if (i_ < s_.size()) return Fail("data after the JSON value");
+ return v;
+ }
+
+ private:
+ std::string_view s_;
+ size_t i_ = 0;
+
+ Result<Value> Fail(const char* msg) {
+ return Result<Value>::Fail(std::string("JSON: ") + msg + " at offset " + std::to_string(i_));
+ }
+ void Space() {
+ while (i_ < s_.size() && (s_[i_] == ' ' || s_[i_] == '\t' || s_[i_] == '\n' || s_[i_] == '\r')) i_++;
+ }
+ bool Literal(std::string_view w) {
+ if (s_.substr(i_, w.size()) == w) {
+ i_ += w.size();
+ return true;
+ }
+ return false;
+ }
+
+ Result<Value> Val(int depth) {
+ if (depth > kMaxDepth) {
+ return Result<Value>::Fail("JSON nested deeper than " + std::to_string(kMaxDepth) + " levels");
+ }
+ if (i_ >= s_.size()) return Fail("unexpected end of text");
+ const char c = s_[i_];
+ if (c == '{') {
+ i_++;
+ return Obj(depth + 1);
+ }
+ if (c == '[') {
+ i_++;
+ return Arr(depth + 1);
+ }
+ if (c == '"') {
+ i_++;
+ std::string out;
+ if (!Str(out)) return Fail(bad_);
+ return Result<Value>::Ok(Value::String(std::move(out)));
+ }
+ if (Literal("true")) return Result<Value>::Ok(Value::Bool(true));
+ if (Literal("false")) return Result<Value>::Ok(Value::Bool(false));
+ if (Literal("null")) return Result<Value>::Ok(Value::Null());
+ if (Literal("NaN")) return Result<Value>::Ok(Value::Float(std::nan("")));
+ if (Literal("Infinity")) return Result<Value>::Ok(Value::Float(HUGE_VAL));
+ if (Literal("-Infinity")) return Result<Value>::Ok(Value::Float(-HUGE_VAL));
+ if (c == '-' || (c >= '0' && c <= '9')) return Num();
+ return Fail("unexpected character");
+ }
+
+ Result<Value> Arr(int depth) {
+ Value list = Value::Array();
+ Space();
+ if (i_ < s_.size() && s_[i_] == ']') {
+ i_++;
+ return Result<Value>::Ok(std::move(list));
+ }
+ for (;;) {
+ Space();
+ Result<Value> v = Val(depth);
+ if (!v.ok()) return v;
+ list.Push(std::move(v).value());
+ Space();
+ if (i_ >= s_.size()) return Fail("unterminated array");
+ if (s_[i_] == ',') {
+ i_++;
+ } else if (s_[i_] == ']') {
+ i_++;
+ return Result<Value>::Ok(std::move(list));
+ } else {
+ return Fail("expected , or ]");
+ }
+ }
+ }
+
+ Result<Value> Obj(int depth) {
+ Value obj = Value::Object();
+ std::unordered_map<std::string, size_t> at;
+ Space();
+ if (i_ < s_.size() && s_[i_] == '}') {
+ i_++;
+ return Result<Value>::Ok(std::move(obj));
+ }
+ for (;;) {
+ Space();
+ if (i_ >= s_.size() || s_[i_] != '"') return Fail("expected a string key");
+ i_++;
+ std::string key;
+ if (!Str(key)) return Fail(bad_);
+ Space();
+ if (i_ >= s_.size() || s_[i_] != ':') return Fail("expected :");
+ i_++;
+ Space();
+ Result<Value> v = Val(depth);
+ if (!v.ok()) return v;
+ obj.Set(std::move(key), std::move(v).value()); // first position, last value
+ Space();
+ if (i_ >= s_.size()) return Fail("unterminated object");
+ if (s_[i_] == ',') {
+ i_++;
+ } else if (s_[i_] == '}') {
+ i_++;
+ return Result<Value>::Ok(std::move(obj));
+ } else {
+ return Fail("expected , or }");
+ }
+ }
+ }
+
+ // Reads a string whose opening quote was consumed. On failure err_ names the problem.
+ bool Str(std::string& out) {
+ while (i_ < s_.size()) {
+ const unsigned char c = static_cast<unsigned char>(s_[i_]);
+ if (c == '"') {
+ i_++;
+ return true;
+ }
+ if (c < 0x20) return Bad("control character in a string");
+ if (c != '\\') {
+ out.push_back(char(c));
+ i_++;
+ continue;
+ }
+ i_++;
+ if (i_ >= s_.size()) break;
+ const char esc = s_[i_++];
+ switch (esc) {
+ case '"': case '\\': case '/': out.push_back(esc); break;
+ case 'b': out.push_back('\b'); break;
+ case 'f': out.push_back('\f'); break;
+ case 'n': out.push_back('\n'); break;
+ case 'r': out.push_back('\r'); break;
+ case 't': out.push_back('\t'); break;
+ case 'u': {
+ uint32_t r;
+ if (!Hex4(r)) return false;
+ if (r >= 0xD800 && r < 0xDC00) { // a high surrogate needs its low half
+ if (s_.substr(i_, 2) != "\\u") return Bad("lone surrogate escape");
+ i_ += 2;
+ uint32_t lo;
+ if (!Hex4(lo)) return false;
+ if (lo < 0xDC00 || lo >= 0xE000) return Bad("lone surrogate escape");
+ r = 0x10000 + ((r - 0xD800) << 10) + (lo - 0xDC00);
+ } else if (r >= 0xDC00 && r < 0xE000) {
+ return Bad("lone surrogate escape");
+ }
+ unicodedata::AppendRune(out, r);
+ break;
+ }
+ default: return Bad("invalid escape");
+ }
+ }
+ return Bad("unterminated string");
+ }
+
+ bool Bad(const char* msg) {
+ bad_ = msg;
+ return false;
+ }
+
+ bool Hex4(uint32_t& r) {
+ if (i_ + 4 > s_.size()) return Bad("truncated \\u escape");
+ r = 0;
+ for (int k = 0; k < 4; k++) {
+ char c = s_[i_ + k];
+ int d;
+ if (c >= '0' && c <= '9') d = c - '0';
+ else if (c >= 'a' && c <= 'f') d = c - 'a' + 10;
+ else if (c >= 'A' && c <= 'F') d = c - 'A' + 10;
+ else return Bad("invalid \\u escape");
+ r = r * 16 + uint32_t(d);
+ }
+ i_ += 4;
+ return true;
+ }
+
+ Result<Value> Num() {
+ const size_t start = i_;
+ if (s_[i_] == '-') i_++;
+ auto digits = [this]() {
+ size_t n = 0;
+ while (i_ < s_.size() && s_[i_] >= '0' && s_[i_] <= '9') {
+ i_++;
+ n++;
+ }
+ return n;
+ };
+ const size_t int_start = i_;
+ if (i_ < s_.size() && s_[i_] == '0') {
+ i_++;
+ } else if (digits() == 0) {
+ return Fail("invalid number");
+ }
+ const size_t int_digits = i_ - int_start;
+ bool is_int = true;
+ if (i_ < s_.size() && s_[i_] == '.') {
+ i_++;
+ if (digits() == 0) return Fail("invalid number");
+ is_int = false;
+ }
+ if (i_ < s_.size() && (s_[i_] == 'e' || s_[i_] == 'E')) {
+ i_++;
+ if (i_ < s_.size() && (s_[i_] == '+' || s_[i_] == '-')) i_++;
+ if (digits() == 0) return Fail("invalid number");
+ is_int = false;
+ }
+ const std::string text(s_.substr(start, i_ - start));
+ if (is_int) {
+ if (int_digits > size_t(kMaxIntDigits)) return Fail("int literal exceeds the limit of 4300 digits");
+ errno = 0;
+ long long n = std::strtoll(text.c_str(), nullptr, 10);
+ if (errno == 0) return Result<Value>::Ok(Value::Int(n));
+ return Result<Value>::Ok(Value::BigInt(text));
+ }
+ // Out of range is +-Inf, as float("1e400") is in Python.
+ return Result<Value>::Ok(Value::Float(std::strtod(text.c_str(), nullptr)));
+ }
+
+ public:
+ const char* bad_ = "string";
+};
+
+} // namespace
+
+Result<Value> Parse(std::string_view text) {
+ Parser p(text);
+ return p.Document();
+}
+
+namespace {
+
+bool WriteString(std::string& b, const std::string& s) {
+ if (!unicodedata::ValidUtf8(s)) return false;
+ b.push_back('"');
+ for (char ch : s) {
+ const unsigned char c = static_cast<unsigned char>(ch);
+ switch (c) {
+ case '"': b += "\\\""; break;
+ case '\\': b += "\\\\"; break;
+ case '\n': b += "\\n"; break;
+ case '\r': b += "\\r"; break;
+ case '\t': b += "\\t"; break;
+ case '\b': b += "\\b"; break;
+ case '\f': b += "\\f"; break;
+ default:
+ if (c < 0x20) {
+ char buf[8];
+ std::snprintf(buf, sizeof buf, "\\u%04x", c);
+ b += buf;
+ } else {
+ b.push_back(ch);
+ }
+ }
+ }
+ b.push_back('"');
+ return true;
+}
+
+std::string FloatStr(double f) {
+ if (std::isnan(f)) return "nan";
+ if (std::isinf(f)) return f > 0 ? "inf" : "-inf";
+ return pyjson::ReprFloat(f);
+}
+
+Result<bool> Write(std::string& b, const Value& v, int depth) {
+ if (depth > kMaxDepth) {
+ return Result<bool>::Fail("value nested deeper than " + std::to_string(kMaxDepth) + " levels");
+ }
+ switch (v.kind()) {
+ case Value::Kind::Null: b += "null"; break;
+ case Value::Kind::Bool: b += v.as_bool() ? "true" : "false"; break;
+ case Value::Kind::Int: b += std::to_string(v.as_int()); break;
+ case Value::Kind::BigInt: b += v.big_int(); break;
+ case Value::Kind::Float: b += pyjson::ReprFloat(v.as_float()); break;
+ case Value::Kind::String:
+ if (!WriteString(b, v.as_string())) {
+ return Result<bool>::Fail("string is not valid UTF-8, which a Python str always is");
+ }
+ break;
+ case Value::Kind::Array: {
+ b.push_back('[');
+ for (size_t i = 0; i < v.size(); i++) {
+ if (i > 0) b += ", ";
+ Result<bool> r = Write(b, v.value(i), depth + 1);
+ if (!r.ok()) return r;
+ }
+ b.push_back(']');
+ break;
+ }
+ case Value::Kind::Object: {
+ b.push_back('{');
+ for (size_t i = 0; i < v.size(); i++) {
+ if (i > 0) b += ", ";
+ if (!WriteString(b, v.key(i))) {
+ return Result<bool>::Fail("string is not valid UTF-8, which a Python str always is");
+ }
+ b += ": ";
+ Result<bool> r = Write(b, v.value(i), depth + 1);
+ if (!r.ok()) return r;
+ }
+ b.push_back('}');
+ break;
+ }
+ }
+ return Result<bool>::Ok(true);
+}
+
+void WriteRepr(std::string& b, const Value& v) {
+ switch (v.kind()) {
+ case Value::Kind::Null: b += "None"; break;
+ case Value::Kind::Bool: b += v.as_bool() ? "True" : "False"; break;
+ case Value::Kind::String: b += pyjson::Repr(v.as_string()); break;
+ case Value::Kind::Int:
+ case Value::Kind::BigInt:
+ case Value::Kind::Float: b += Str(v); break;
+ case Value::Kind::Array:
+ b.push_back('[');
+ for (size_t i = 0; i < v.size(); i++) {
+ if (i > 0) b += ", ";
+ WriteRepr(b, v.value(i));
+ }
+ b.push_back(']');
+ break;
+ case Value::Kind::Object:
+ b.push_back('{');
+ for (size_t i = 0; i < v.size(); i++) {
+ if (i > 0) b += ", ";
+ b += pyjson::Repr(v.key(i));
+ b += ": ";
+ WriteRepr(b, v.value(i));
+ }
+ b.push_back('}');
+ break;
+ }
+}
+
+} // namespace
+
+Result<std::string> Dumps(const Value& v) {
+ std::string out;
+ Result<bool> r = Write(out, v, 0);
+ if (!r.ok()) return Result<std::string>::Fail(r.error());
+ return Result<std::string>::Ok(std::move(out));
+}
+
+std::string TypeName(const Value& v) {
+ switch (v.kind()) {
+ case Value::Kind::Null: return "NoneType";
+ case Value::Kind::Bool: return "bool";
+ case Value::Kind::Int:
+ case Value::Kind::BigInt: return "int";
+ case Value::Kind::Float: return "float";
+ case Value::Kind::String: return "str";
+ case Value::Kind::Array: return "list";
+ case Value::Kind::Object: return "dict";
+ }
+ return "";
+}
+
+std::string Repr(const Value& v) {
+ std::string out;
+ WriteRepr(out, v);
+ return out;
+}
+
+std::string Str(const Value& v) {
+ switch (v.kind()) {
+ case Value::Kind::String: return v.as_string();
+ case Value::Kind::Int: return std::to_string(v.as_int());
+ case Value::Kind::BigInt: return v.big_int();
+ case Value::Kind::Float: return FloatStr(v.as_float());
+ default: return Repr(v);
+ }
+}
+
+} // namespace pyvalue
+} // namespace laya
diff --git a/laya-cpp/src/sha256.cpp b/laya-cpp/src/sha256.cpp
new file mode 100644
index 00000000..97089cd8
--- /dev/null
+++ b/laya-cpp/src/sha256.cpp
@@ -0,0 +1,115 @@
+#include "laya/sha256.hpp"
+
+#include <cstring>
+
+namespace laya {
+namespace {
+
+constexpr uint32_t kK[64] = {
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
+
+inline uint32_t Rotr(uint32_t x, int n) { return (x >> n) | (x << (32 - n)); }
+
+} // namespace
+
+Sha256::Sha256()
+ : h_{0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19},
+ buf_{} {}
+
+void Sha256::Block(const uint8_t* p) {
+ uint32_t w[64];
+ for (int i = 0; i < 16; i++) { // big-endian words
+ w[i] = (uint32_t(p[4 * i]) << 24) | (uint32_t(p[4 * i + 1]) << 16) | (uint32_t(p[4 * i + 2]) << 8) |
+ uint32_t(p[4 * i + 3]);
+ }
+ for (int i = 16; i < 64; i++) {
+ uint32_t s0 = Rotr(w[i - 15], 7) ^ Rotr(w[i - 15], 18) ^ (w[i - 15] >> 3);
+ uint32_t s1 = Rotr(w[i - 2], 17) ^ Rotr(w[i - 2], 19) ^ (w[i - 2] >> 10);
+ w[i] = w[i - 16] + s0 + w[i - 7] + s1;
+ }
+ uint32_t a = h_[0], b = h_[1], c = h_[2], d = h_[3], e = h_[4], f = h_[5], g = h_[6], h = h_[7];
+ for (int i = 0; i < 64; i++) {
+ uint32_t t1 = h + (Rotr(e, 6) ^ Rotr(e, 11) ^ Rotr(e, 25)) + ((e & f) ^ (~e & g)) + kK[i] + w[i];
+ uint32_t t2 = (Rotr(a, 2) ^ Rotr(a, 13) ^ Rotr(a, 22)) + ((a & b) ^ (a & c) ^ (b & c));
+ h = g;
+ g = f;
+ f = e;
+ e = d + t1;
+ d = c;
+ c = b;
+ b = a;
+ a = t1 + t2;
+ }
+ h_[0] += a;
+ h_[1] += b;
+ h_[2] += c;
+ h_[3] += d;
+ h_[4] += e;
+ h_[5] += f;
+ h_[6] += g;
+ h_[7] += h;
+}
+
+void Sha256::Update(const void* data, size_t size) {
+ const uint8_t* p = static_cast<const uint8_t*>(data);
+ total_ += size;
+ while (size > 0) {
+ size_t take = 64 - used_;
+ if (take > size) take = size;
+ std::memcpy(buf_ + used_, p, take);
+ used_ += take;
+ p += take;
+ size -= take;
+ if (used_ == 64) {
+ Block(buf_);
+ used_ = 0;
+ }
+ }
+}
+
+std::array<uint8_t, 32> Sha256::Finish() {
+ const uint64_t bits = total_ * 8;
+ uint8_t pad[72] = {0x80};
+ // Pad with 0x80 then zeros to 56 mod 64, then the 64-bit big-endian bit length.
+ size_t padlen = (used_ < 56) ? (56 - used_) : (120 - used_);
+ uint8_t len[8];
+ for (int i = 0; i < 8; i++) len[i] = uint8_t(bits >> (56 - 8 * i));
+ Update(pad, padlen);
+ Update(len, 8);
+ std::array<uint8_t, 32> out{};
+ for (int i = 0; i < 8; i++) {
+ out[4 * i] = uint8_t(h_[i] >> 24);
+ out[4 * i + 1] = uint8_t(h_[i] >> 16);
+ out[4 * i + 2] = uint8_t(h_[i] >> 8);
+ out[4 * i + 3] = uint8_t(h_[i]);
+ }
+ return out;
+}
+
+std::array<uint8_t, 32> Sha256Of(std::string_view data) {
+ Sha256 s;
+ s.Update(data);
+ return s.Finish();
+}
+
+std::string ToHex(const std::array<uint8_t, 32>& digest) {
+ static const char* kHex = "0123456789abcdef";
+ std::string out;
+ out.reserve(64);
+ for (uint8_t b : digest) {
+ out.push_back(kHex[b >> 4]);
+ out.push_back(kHex[b & 15]);
+ }
+ return out;
+}
+
+std::string Sha256Hex(std::string_view data) { return ToHex(Sha256Of(data)); }
+
+} // namespace laya
diff --git a/laya-cpp/src/unicodedata.cpp b/laya-cpp/src/unicodedata.cpp
new file mode 100644
index 00000000..1d844246
--- /dev/null
+++ b/laya-cpp/src/unicodedata.cpp
@@ -0,0 +1,333 @@
+#include "laya/unicodedata.hpp"
+
+#include <cstddef>
+
+namespace laya {
+namespace unicodedata {
+namespace {
+#include "unicodedata_tables.inc"
+
+template <size_t N>
+bool Contains(const uint32_t (&pairs)[N], char32_t r) {
+ size_t lo = 0, hi = N / 2; // half-open over pairs
+ while (lo < hi) {
+ size_t mid = lo + (hi - lo) / 2;
+ if (r < pairs[mid * 2]) {
+ hi = mid;
+ } else if (r > pairs[mid * 2 + 1]) {
+ lo = mid + 1;
+ } else {
+ return true;
+ }
+ }
+ return false;
+}
+
+template <size_t N>
+bool Lookup(const uint64_t (&mask)[2], const uint32_t (&pairs)[N], char32_t r) {
+ if (r < 0x80) return ((mask[r >> 6] >> (r & 63)) & 1) != 0;
+ return Contains(pairs, r);
+}
+
+bool IsCased(char32_t r) { return IsUpper(r) || IsLower(r) || Contains(k_title, r); }
+
+constexpr char32_t kCapitalSigma = 0x03A3, kSmallSigma = 0x03C3, kFinalSigma = 0x03C2;
+
+bool SigmaIsFinal(std::string_view s, size_t i) {
+ bool before = false;
+ for (size_t j = i; j > 0;) {
+ Decoded d = DecodeLastRune(s.substr(0, j));
+ j -= d.width;
+ if (Contains(k_case_ignorable, d.rune)) continue;
+ before = Contains(k_cased, d.rune);
+ break;
+ }
+ if (!before) return false;
+ for (size_t j = i + 2; j < s.size();) { // U+03A3 is two bytes in UTF-8
+ Decoded d = DecodeRune(s.substr(j));
+ j += d.width;
+ if (Contains(k_case_ignorable, d.rune)) continue;
+ return !Contains(k_cased, d.rune);
+ }
+ return true;
+}
+
+size_t InvalidPrefixLen(std::string_view b) {
+ uint8_t lo = 0x80, hi = 0xBF;
+ size_t need;
+ uint8_t c = uint8_t(b[0]);
+ if (c >= 0xC2 && c <= 0xDF) {
+ need = 1;
+ } else if (c == 0xE0) {
+ need = 2;
+ lo = 0xA0;
+ } else if ((c >= 0xE1 && c <= 0xEC) || c == 0xEE || c == 0xEF) {
+ need = 2;
+ } else if (c == 0xED) {
+ need = 2;
+ hi = 0x9F;
+ } else if (c == 0xF0) {
+ need = 3;
+ lo = 0x90;
+ } else if (c >= 0xF1 && c <= 0xF3) {
+ need = 3;
+ } else if (c == 0xF4) {
+ need = 3;
+ hi = 0x8F;
+ } else {
+ return 1;
+ }
+ size_t n = 1;
+ for (; n <= need && n < b.size(); n++) {
+ uint8_t x = uint8_t(b[n]);
+ if (x < lo || x > hi) break;
+ lo = 0x80;
+ hi = 0xBF;
+ }
+ return n;
+}
+
+} // namespace
+
+const char* const kVersion = kUnicodeVersion;
+
+bool IsAlpha(char32_t r) { return Lookup(k_alpha_ascii, k_alpha, r); }
+bool IsWord(char32_t r) { return Lookup(k_word_ascii, k_word, r); }
+bool IsCombining(char32_t r) { return Lookup(k_combining_ascii, k_combining, r); }
+bool IsDigit(char32_t r) { return Lookup(k_digit_ascii, k_digit, r); }
+bool IsUpper(char32_t r) { return Lookup(k_upper_ascii, k_upper, r); }
+bool IsLower(char32_t r) { return Lookup(k_lower_ascii, k_lower, r); }
+bool IsSpace(char32_t r) { return Lookup(k_space_ascii, k_space, r); }
+bool IsMark(char32_t r) { return Lookup(k_mark_ascii, k_mark, r); }
+bool IsInitial(char32_t r) { return Lookup(k_initial_ascii, k_initial, r); }
+bool IsPrintable(char32_t r) { return Lookup(k_printable_ascii, k_printable, r); }
+bool IsPythonWord(char32_t r) { return r == '_' || IsWord(r) || IsDigit(r); }
+bool IsWordOrDigit(char32_t r) { return IsWord(r) || IsDigit(r); }
+
+bool IsUpperString(std::string_view s) {
+ bool saw = false;
+ for (size_t i = 0; i < s.size();) {
+ Decoded d = DecodeRune(s.substr(i));
+ i += d.width;
+ if (!IsCased(d.rune)) continue;
+ if (!IsUpper(d.rune)) return false;
+ saw = true;
+ }
+ return saw;
+}
+
+Decoded DecodeRune(std::string_view s) {
+ const uint8_t c = uint8_t(s[0]);
+ if (c < 0x80) return {c, 1};
+ size_t need;
+ uint8_t lo = 0x80, hi = 0xBF;
+ char32_t r;
+ if (c >= 0xC2 && c <= 0xDF) {
+ need = 1;
+ r = c & 0x1F;
+ } else if (c >= 0xE0 && c <= 0xEF) {
+ need = 2;
+ r = c & 0x0F;
+ if (c == 0xE0) lo = 0xA0;
+ if (c == 0xED) hi = 0x9F;
+ } else if (c >= 0xF0 && c <= 0xF4) {
+ need = 3;
+ r = c & 0x07;
+ if (c == 0xF0) lo = 0x90;
+ if (c == 0xF4) hi = 0x8F;
+ } else {
+ return {kReplacement, 1};
+ }
+ if (s.size() < need + 1) return {kReplacement, 1};
+ for (size_t i = 1; i <= need; i++) {
+ uint8_t x = uint8_t(s[i]);
+ if (x < lo || x > hi) return {kReplacement, 1};
+ lo = 0x80;
+ hi = 0xBF;
+ r = (r << 6) | (x & 0x3F);
+ }
+ return {r, need + 1};
+}
+
+Decoded DecodeLastRune(std::string_view s) {
+ const size_t end = s.size();
+ if (uint8_t(s[end - 1]) < 0x80) return {uint8_t(s[end - 1]), 1};
+ const size_t lim = end > 4 ? end - 4 : 0;
+ size_t start = end - 1;
+ while (start > lim && (uint8_t(s[start]) & 0xC0) == 0x80) start--; // back to a start byte
+ Decoded d = DecodeRune(s.substr(start, end - start));
+ if (start + d.width != end) return {kReplacement, 1};
+ return d;
+}
+
+bool ValidUtf8(std::string_view s) {
+ for (size_t i = 0; i < s.size();) {
+ if (uint8_t(s[i]) < 0x80) {
+ i++;
+ continue;
+ }
+ Decoded d = DecodeRune(s.substr(i));
+ if (d.rune == kReplacement && d.width == 1) return false;
+ i += d.width;
+ }
+ return true;
+}
+
+void AppendRune(std::string& out, char32_t r) {
+ if (r > 0x10FFFF || (r >= 0xD800 && r <= 0xDFFF)) r = kReplacement;
+ if (r < 0x80) {
+ out.push_back(char(r));
+ } else if (r < 0x800) {
+ out.push_back(char(0xC0 | (r >> 6)));
+ out.push_back(char(0x80 | (r & 0x3F)));
+ } else if (r < 0x10000) {
+ out.push_back(char(0xE0 | (r >> 12)));
+ out.push_back(char(0x80 | ((r >> 6) & 0x3F)));
+ out.push_back(char(0x80 | (r & 0x3F)));
+ } else {
+ out.push_back(char(0xF0 | (r >> 18)));
+ out.push_back(char(0x80 | ((r >> 12) & 0x3F)));
+ out.push_back(char(0x80 | ((r >> 6) & 0x3F)));
+ out.push_back(char(0x80 | (r & 0x3F)));
+ }
+}
+
+size_t RuneCount(std::string_view s) {
+ size_t n = 0;
+ for (size_t i = 0; i < s.size(); n++) i += DecodeRune(s.substr(i)).width;
+ return n;
+}
+
+char32_t LowerRune(char32_t r) {
+ if (r < 0x80) return (r >= 'A' && r <= 'Z') ? r + 32 : r;
+ size_t lo = 0, hi = sizeof(k_lower_from) / sizeof(k_lower_from[0]);
+ while (lo < hi) {
+ size_t mid = lo + (hi - lo) / 2;
+ if (r < k_lower_from[mid]) {
+ hi = mid;
+ } else if (r > k_lower_from[mid]) {
+ lo = mid + 1;
+ } else {
+ return k_lower_to[mid];
+ }
+ }
+ return r;
+}
+
+std::string Lower(std::string_view s) {
+ std::string out;
+ out.reserve(s.size());
+ for (size_t i = 0; i < s.size();) {
+ Decoded d = DecodeRune(s.substr(i));
+ if (d.rune == kCapitalSigma) {
+ AppendRune(out, SigmaIsFinal(s, i) ? kFinalSigma : kSmallSigma);
+ } else {
+ bool multi = false;
+ for (const LowerMulti& m : k_lower_multi) {
+ if (m.from == d.rune) {
+ for (uint32_t k = 0; k < m.count; k++) AppendRune(out, m.to[k]);
+ multi = true;
+ break;
+ }
+ }
+ if (!multi) AppendRune(out, LowerRune(d.rune));
+ }
+ i += d.width;
+ }
+ return out;
+}
+
+std::string Lossy(std::string_view b) {
+ std::string out;
+ out.reserve(b.size() + 8);
+ for (size_t i = 0; i < b.size();) {
+ Decoded d = DecodeRune(b.substr(i));
+ if (!(d.rune == kReplacement && d.width == 1)) {
+ out.append(b.substr(i, d.width));
+ i += d.width;
+ continue;
+ }
+ AppendRune(out, kReplacement);
+ i += InvalidPrefixLen(b.substr(i));
+ }
+ return out;
+}
+
+std::string ToValid(std::string_view s) { return Lossy(s); }
+
+std::string DecimalASCII(std::string_view s) {
+ std::string out;
+ for (size_t i = 0; i < s.size();) {
+ Decoded d = DecodeRune(s.substr(i));
+ i += d.width;
+ char32_t r = d.rune;
+ if (r >= 0x80 && Contains(k_digit, r)) {
+ size_t lo = 0, hi = sizeof(k_digit) / sizeof(k_digit[0]) / 2 - 1;
+ while (lo < hi) { // last range starting at or before r
+ size_t mid = (lo + hi + 1) / 2;
+ if (k_digit[mid * 2] <= r) {
+ lo = mid;
+ } else {
+ hi = mid - 1;
+ }
+ }
+ r = '0' + (r - k_digit[lo * 2]) % 10;
+ }
+ AppendRune(out, r);
+ }
+ return out;
+}
+
+namespace {
+std::string TrimBy(std::string_view s, bool numeric) {
+ auto trims = [numeric](char32_t r) { return IsSpace(r) && !(numeric && r >= 0x1c && r <= 0x1f); };
+ size_t b = 0, e = s.size();
+ while (b < e) {
+ Decoded d = DecodeRune(s.substr(b, e - b));
+ if (!trims(d.rune)) break;
+ b += d.width;
+ }
+ while (e > b) {
+ Decoded d = DecodeLastRune(s.substr(b, e - b));
+ if (!trims(d.rune)) break;
+ e -= d.width;
+ }
+ return std::string(s.substr(b, e - b));
+}
+} // namespace
+
+std::string Strip(std::string_view s) { return TrimBy(s, false); }
+std::string StripNumeric(std::string_view s) { return TrimBy(s, true); }
+
+bool IsBlank(std::string_view s) {
+ for (size_t i = 0; i < s.size();) {
+ Decoded d = DecodeRune(s.substr(i));
+ if (!IsSpace(d.rune)) return false;
+ i += d.width;
+ }
+ return true;
+}
+
+std::vector<std::string> Fields(std::string_view s) {
+ std::vector<std::string> out;
+ bool in = false;
+ size_t start = 0;
+ for (size_t i = 0; i < s.size();) {
+ Decoded d = DecodeRune(s.substr(i));
+ if (IsSpace(d.rune)) {
+ if (in) {
+ out.emplace_back(s.substr(start, i - start));
+ in = false;
+ }
+ } else if (!in) {
+ in = true;
+ start = i;
+ }
+ i += d.width;
+ }
+ if (in) out.emplace_back(s.substr(start));
+ return out;
+}
+
+} // namespace unicodedata
+} // namespace laya
diff --git a/laya-cpp/src/unicodedata_tables.inc b/laya-cpp/src/unicodedata_tables.inc
new file mode 100644
index 00000000..3dea8f7f
--- /dev/null
+++ b/laya-cpp/src/unicodedata_tables.inc
@@ -0,0 +1,1069 @@
+// Code generated by laya-go/scripts/gen_tables.py --lang cpp from CPython 3.12 (Unicode 15.0.0); DO NOT EDIT.
+//
+// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp
+
+inline constexpr const char* kUnicodeVersion = "15.0.0";
+
+inline constexpr uint32_t k_alpha[] = {
+ 65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736,
+ 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, 906, 908, 908,
+ 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1488, 1514,
+ 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788,
+ 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042,
+ 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190,
+ 2208, 2249, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448,
+ 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529,
+ 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614,
+ 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736,
+ 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832,
+ 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929,
+ 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980,
+ 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133,
+ 3160, 3162, 3165, 3165, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251,
+ 3253, 3257, 3261, 3261, 3293, 3294, 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386,
+ 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515,
+ 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722,
+ 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807,
+ 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189,
+ 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301,
+ 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749,
+ 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885,
+ 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866,
+ 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103,
+ 6108, 6108, 6176, 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509,
+ 6512, 6516, 6528, 6571, 6576, 6601, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6988,
+ 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7304, 7312, 7354,
+ 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965,
+ 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116,
+ 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180,
+ 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469,
+ 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521,
+ 8526, 8526, 8579, 8580, 11264, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559,
+ 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696,
+ 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823,
+ 12293, 12294, 12337, 12341, 12347, 12348, 12353, 12438, 12445, 12447, 12449, 12538, 12540,
+ 12543, 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903, 19968, 42124,
+ 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653, 42656,
+ 42725, 42775, 42783, 42786, 42888, 42891, 42954, 42960, 42961, 42963, 42963, 42965, 42969,
+ 42994, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250,
+ 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442,
+ 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588,
+ 43595, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709,
+ 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785,
+ 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881, 43888, 44002,
+ 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275,
+ 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321,
+ 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136,
+ 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487,
+ 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599,
+ 65613, 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66368,
+ 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66560, 66717, 66736,
+ 66771, 66776, 66811, 66816, 66855, 66864, 66915, 66928, 66938, 66940, 66954, 66956, 66962,
+ 66964, 66965, 66967, 66977, 66979, 66993, 66995, 67001, 67003, 67004, 67072, 67382, 67392,
+ 67413, 67424, 67431, 67456, 67461, 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592,
+ 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808,
+ 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096,
+ 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295, 68297,
+ 68324, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786,
+ 68800, 68850, 68864, 68899, 69248, 69289, 69296, 69297, 69376, 69404, 69415, 69415, 69424,
+ 69445, 69488, 69505, 69552, 69572, 69600, 69622, 69635, 69687, 69745, 69746, 69749, 69749,
+ 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69959, 69959, 69968, 70002, 70006,
+ 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187,
+ 70207, 70208, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320,
+ 70366, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457,
+ 70461, 70461, 70480, 70480, 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70753, 70784,
+ 70831, 70852, 70853, 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236,
+ 71296, 71338, 71352, 71352, 71424, 71450, 71488, 71494, 71680, 71723, 71840, 71903, 71935,
+ 71942, 71945, 71945, 71948, 71955, 71957, 71958, 71960, 71983, 71999, 71999, 72001, 72001,
+ 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250,
+ 72250, 72272, 72272, 72284, 72329, 72349, 72349, 72368, 72440, 72704, 72712, 72714, 72750,
+ 72768, 72768, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056,
+ 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488,
+ 73490, 73523, 73648, 73648, 73728, 74649, 74880, 75075, 77712, 77808, 77824, 78895, 78913,
+ 78918, 82944, 83526, 92160, 92728, 92736, 92766, 92784, 92862, 92880, 92909, 92928, 92975,
+ 92992, 92995, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026, 94032, 94032, 94099,
+ 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101589, 101632, 101640, 110576,
+ 110579, 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898, 110928, 110930, 110933,
+ 110933, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808,
+ 113817, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977,
+ 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077,
+ 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138,
+ 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598,
+ 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772,
+ 120779, 122624, 122654, 122661, 122666, 122928, 122989, 123136, 123180, 123191, 123197, 123214,
+ 123214, 123536, 123565, 123584, 123627, 124112, 124139, 124896, 124902, 124904, 124907, 124909,
+ 124910, 124912, 124926, 124928, 125124, 125184, 125251, 125259, 125259, 126464, 126467, 126469,
+ 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521,
+ 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541,
+ 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557,
+ 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580,
+ 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629,
+ 126633, 126635, 126651, 131072, 173791, 173824, 177977, 177984, 178205, 178208, 183969, 183984,
+ 191456, 194560, 195101, 196608, 201546, 201552, 205743,
+};
+
+inline constexpr uint64_t k_alpha_ascii[2] = {0x0, 0x7fffffe07fffffe};
+
+inline constexpr uint32_t k_word[] = {
+ 65, 90, 97, 122, 170, 170, 178, 179, 181, 181, 185, 186, 188, 190, 192, 214, 216, 246, 248,
+ 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902,
+ 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376,
+ 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774,
+ 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036,
+ 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2160,
+ 2183, 2185, 2190, 2208, 2249, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437,
+ 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524,
+ 2525, 2527, 2529, 2544, 2545, 2548, 2553, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602,
+ 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703,
+ 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2809,
+ 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908,
+ 2909, 2911, 2913, 2929, 2935, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972,
+ 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3056, 3058, 3077, 3084, 3086,
+ 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169, 3192, 3198, 3200,
+ 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3293, 3294, 3296,
+ 3297, 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3416,
+ 3425, 3440, 3448, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585,
+ 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751,
+ 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3882, 3891, 3904,
+ 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197,
+ 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348,
+ 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786,
+ 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4969,
+ 4988, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870,
+ 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108,
+ 6108, 6128, 6137, 6176, 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480,
+ 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6618, 6618, 6656, 6678, 6688, 6740, 6823, 6823, 6917,
+ 6963, 6981, 6988, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296,
+ 7304, 7312, 7354, 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680,
+ 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031,
+ 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160,
+ 8172, 8178, 8180, 8182, 8188, 8304, 8305, 8308, 8313, 8319, 8329, 8336, 8348, 8450, 8450, 8455,
+ 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495,
+ 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8528, 8585, 9312, 9371, 9450, 9471, 10102, 10131,
+ 11264, 11492, 11499, 11502, 11506, 11507, 11517, 11517, 11520, 11557, 11559, 11559, 11565,
+ 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702,
+ 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823, 12293,
+ 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12445, 12447, 12449, 12538,
+ 12540, 12543, 12549, 12591, 12593, 12686, 12690, 12693, 12704, 12735, 12784, 12799, 12832,
+ 12841, 12872, 12879, 12881, 12895, 12928, 12937, 12977, 12991, 13312, 19903, 19968, 42124,
+ 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653, 42656,
+ 42735, 42775, 42783, 42786, 42888, 42891, 42954, 42960, 42961, 42963, 42963, 42965, 42969,
+ 42994, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43056, 43061, 43072, 43123, 43138,
+ 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388,
+ 43396, 43442, 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584,
+ 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702,
+ 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777,
+ 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43881,
+ 43888, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256,
+ 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318,
+ 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008,
+ 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479,
+ 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596,
+ 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65799, 65843, 65856, 65912, 65930, 65931,
+ 66176, 66204, 66208, 66256, 66273, 66299, 66304, 66339, 66349, 66378, 66384, 66421, 66432,
+ 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66736, 66771, 66776, 66811,
+ 66816, 66855, 66864, 66915, 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967,
+ 66977, 66979, 66993, 66995, 67001, 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431,
+ 67456, 67461, 67463, 67504, 67506, 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639,
+ 67640, 67644, 67644, 67647, 67669, 67672, 67702, 67705, 67742, 67751, 67759, 67808, 67826,
+ 67828, 67829, 67835, 67867, 67872, 67897, 67968, 68023, 68028, 68047, 68050, 68096, 68112,
+ 68115, 68117, 68119, 68121, 68149, 68160, 68168, 68192, 68222, 68224, 68255, 68288, 68295,
+ 68297, 68324, 68331, 68335, 68352, 68405, 68416, 68437, 68440, 68466, 68472, 68497, 68521,
+ 68527, 68608, 68680, 68736, 68786, 68800, 68850, 68858, 68899, 69216, 69246, 69248, 69289,
+ 69296, 69297, 69376, 69415, 69424, 69445, 69457, 69460, 69488, 69505, 69552, 69579, 69600,
+ 69622, 69635, 69687, 69714, 69733, 69745, 69746, 69749, 69749, 69763, 69807, 69840, 69864,
+ 69891, 69926, 69956, 69956, 69959, 69959, 69968, 70002, 70006, 70006, 70019, 70066, 70081,
+ 70084, 70106, 70106, 70108, 70108, 70113, 70132, 70144, 70161, 70163, 70187, 70207, 70208,
+ 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405,
+ 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461,
+ 70480, 70480, 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, 70852,
+ 70853, 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338,
+ 71352, 71352, 71424, 71450, 71482, 71483, 71488, 71494, 71680, 71723, 71840, 71903, 71914,
+ 71922, 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958, 71960, 71983, 71999, 71999,
+ 72001, 72001, 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163, 72192, 72192, 72203,
+ 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349, 72349, 72368, 72440, 72704, 72712,
+ 72714, 72750, 72768, 72768, 72794, 72812, 72818, 72847, 72960, 72966, 72968, 72969, 72971,
+ 73008, 73030, 73030, 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458,
+ 73474, 73474, 73476, 73488, 73490, 73523, 73648, 73648, 73664, 73684, 73728, 74649, 74752,
+ 74862, 74880, 75075, 77712, 77808, 77824, 78895, 78913, 78918, 82944, 83526, 92160, 92728,
+ 92736, 92766, 92784, 92862, 92880, 92909, 92928, 92975, 92992, 92995, 93019, 93025, 93027,
+ 93047, 93053, 93071, 93760, 93846, 93952, 94026, 94032, 94032, 94099, 94111, 94176, 94177,
+ 94179, 94179, 94208, 100343, 100352, 101589, 101632, 101640, 110576, 110579, 110581, 110587,
+ 110589, 110590, 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951,
+ 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 119488, 119507,
+ 119520, 119539, 119648, 119672, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970,
+ 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069,
+ 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132,
+ 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570,
+ 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744,
+ 120746, 120770, 120772, 120779, 122624, 122654, 122661, 122666, 122928, 122989, 123136, 123180,
+ 123191, 123197, 123214, 123214, 123536, 123565, 123584, 123627, 124112, 124139, 124896, 124902,
+ 124904, 124907, 124909, 124910, 124912, 124926, 124928, 125124, 125127, 125135, 125184, 125251,
+ 125259, 125259, 126065, 126123, 126125, 126127, 126129, 126132, 126209, 126253, 126255, 126269,
+ 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514,
+ 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537,
+ 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553,
+ 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570,
+ 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619,
+ 126625, 126627, 126629, 126633, 126635, 126651, 127232, 127244, 131072, 173791, 173824, 177977,
+ 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101, 196608, 201546, 201552, 205743,
+};
+
+inline constexpr uint64_t k_word_ascii[2] = {0x0, 0x7fffffe07fffffe};
+
+inline constexpr uint32_t k_combining[] = {
+ 768, 846, 848, 879, 1155, 1159, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479,
+ 1552, 1562, 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770, 1773, 1809, 1809,
+ 1840, 1866, 2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083, 2085, 2087, 2089, 2093, 2137, 2139,
+ 2200, 2207, 2250, 2273, 2275, 2303, 2364, 2364, 2381, 2381, 2385, 2388, 2492, 2492, 2509, 2509,
+ 2558, 2558, 2620, 2620, 2637, 2637, 2748, 2748, 2765, 2765, 2876, 2876, 2893, 2893, 3021, 3021,
+ 3132, 3132, 3149, 3149, 3157, 3158, 3260, 3260, 3277, 3277, 3387, 3388, 3405, 3405, 3530, 3530,
+ 3640, 3642, 3656, 3659, 3768, 3770, 3784, 3787, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897,
+ 3953, 3954, 3956, 3956, 3962, 3965, 3968, 3968, 3970, 3972, 3974, 3975, 4038, 4038, 4151, 4151,
+ 4153, 4154, 4237, 4237, 4957, 4959, 5908, 5909, 5940, 5940, 6098, 6098, 6109, 6109, 6313, 6313,
+ 6457, 6459, 6679, 6680, 6752, 6752, 6773, 6780, 6783, 6783, 6832, 6845, 6847, 6862, 6964, 6964,
+ 6980, 6980, 7019, 7027, 7082, 7083, 7142, 7142, 7154, 7155, 7223, 7223, 7376, 7378, 7380, 7392,
+ 7394, 7400, 7405, 7405, 7412, 7412, 7416, 7417, 7616, 7679, 8400, 8412, 8417, 8417, 8421, 8432,
+ 11503, 11505, 11647, 11647, 11744, 11775, 12330, 12335, 12441, 12442, 42607, 42607, 42612,
+ 42621, 42654, 42655, 42736, 42737, 43014, 43014, 43052, 43052, 43204, 43204, 43232, 43249,
+ 43307, 43309, 43347, 43347, 43443, 43443, 43456, 43456, 43696, 43696, 43698, 43700, 43703,
+ 43704, 43710, 43711, 43713, 43713, 43766, 43766, 44013, 44013, 64286, 64286, 65056, 65071,
+ 66045, 66045, 66272, 66272, 66422, 66426, 68109, 68109, 68111, 68111, 68152, 68154, 68159,
+ 68159, 68325, 68326, 68900, 68903, 69291, 69292, 69373, 69375, 69446, 69456, 69506, 69509,
+ 69702, 69702, 69744, 69744, 69759, 69759, 69817, 69818, 69888, 69890, 69939, 69940, 70003,
+ 70003, 70080, 70080, 70090, 70090, 70197, 70198, 70377, 70378, 70459, 70460, 70477, 70477,
+ 70502, 70508, 70512, 70516, 70722, 70722, 70726, 70726, 70750, 70750, 70850, 70851, 71103,
+ 71104, 71231, 71231, 71350, 71351, 71467, 71467, 71737, 71738, 71997, 71998, 72003, 72003,
+ 72160, 72160, 72244, 72244, 72263, 72263, 72345, 72345, 72767, 72767, 73026, 73026, 73028,
+ 73029, 73111, 73111, 73537, 73538, 92912, 92916, 92976, 92982, 94192, 94193, 113822, 113822,
+ 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364,
+ 122880, 122886, 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123023, 123023,
+ 123184, 123190, 123566, 123566, 123628, 123631, 124140, 124143, 125136, 125142, 125252, 125258,
+};
+
+inline constexpr uint64_t k_combining_ascii[2] = {0x0, 0x0};
+
+inline constexpr uint32_t k_digit[] = {
+ 48, 57, 1632, 1641, 1776, 1785, 1984, 1993, 2406, 2415, 2534, 2543, 2662, 2671, 2790, 2799,
+ 2918, 2927, 3046, 3055, 3174, 3183, 3302, 3311, 3430, 3439, 3558, 3567, 3664, 3673, 3792, 3801,
+ 3872, 3881, 4160, 4169, 4240, 4249, 6112, 6121, 6160, 6169, 6470, 6479, 6608, 6617, 6784, 6793,
+ 6800, 6809, 6992, 7001, 7088, 7097, 7232, 7241, 7248, 7257, 42528, 42537, 43216, 43225, 43264,
+ 43273, 43472, 43481, 43504, 43513, 43600, 43609, 44016, 44025, 65296, 65305, 66720, 66729,
+ 68912, 68921, 69734, 69743, 69872, 69881, 69942, 69951, 70096, 70105, 70384, 70393, 70736,
+ 70745, 70864, 70873, 71248, 71257, 71360, 71369, 71472, 71481, 71904, 71913, 72016, 72025,
+ 72784, 72793, 73040, 73049, 73120, 73129, 73552, 73561, 92768, 92777, 92864, 92873, 93008,
+ 93017, 120782, 120831, 123200, 123209, 123632, 123641, 124144, 124153, 125264, 125273, 130032,
+ 130041,
+};
+
+inline constexpr uint64_t k_digit_ascii[2] = {0x3ff000000000000, 0x0};
+
+inline constexpr uint32_t k_upper[] = {
+ 65, 90, 192, 214, 216, 222, 256, 256, 258, 258, 260, 260, 262, 262, 264, 264, 266, 266, 268,
+ 268, 270, 270, 272, 272, 274, 274, 276, 276, 278, 278, 280, 280, 282, 282, 284, 284, 286, 286,
+ 288, 288, 290, 290, 292, 292, 294, 294, 296, 296, 298, 298, 300, 300, 302, 302, 304, 304, 306,
+ 306, 308, 308, 310, 310, 313, 313, 315, 315, 317, 317, 319, 319, 321, 321, 323, 323, 325, 325,
+ 327, 327, 330, 330, 332, 332, 334, 334, 336, 336, 338, 338, 340, 340, 342, 342, 344, 344, 346,
+ 346, 348, 348, 350, 350, 352, 352, 354, 354, 356, 356, 358, 358, 360, 360, 362, 362, 364, 364,
+ 366, 366, 368, 368, 370, 370, 372, 372, 374, 374, 376, 377, 379, 379, 381, 381, 385, 386, 388,
+ 388, 390, 391, 393, 395, 398, 401, 403, 404, 406, 408, 412, 413, 415, 416, 418, 418, 420, 420,
+ 422, 423, 425, 425, 428, 428, 430, 431, 433, 435, 437, 437, 439, 440, 444, 444, 452, 452, 455,
+ 455, 458, 458, 461, 461, 463, 463, 465, 465, 467, 467, 469, 469, 471, 471, 473, 473, 475, 475,
+ 478, 478, 480, 480, 482, 482, 484, 484, 486, 486, 488, 488, 490, 490, 492, 492, 494, 494, 497,
+ 497, 500, 500, 502, 504, 506, 506, 508, 508, 510, 510, 512, 512, 514, 514, 516, 516, 518, 518,
+ 520, 520, 522, 522, 524, 524, 526, 526, 528, 528, 530, 530, 532, 532, 534, 534, 536, 536, 538,
+ 538, 540, 540, 542, 542, 544, 544, 546, 546, 548, 548, 550, 550, 552, 552, 554, 554, 556, 556,
+ 558, 558, 560, 560, 562, 562, 570, 571, 573, 574, 577, 577, 579, 582, 584, 584, 586, 586, 588,
+ 588, 590, 590, 880, 880, 882, 882, 886, 886, 895, 895, 902, 902, 904, 906, 908, 908, 910, 911,
+ 913, 929, 931, 939, 975, 975, 978, 980, 984, 984, 986, 986, 988, 988, 990, 990, 992, 992, 994,
+ 994, 996, 996, 998, 998, 1000, 1000, 1002, 1002, 1004, 1004, 1006, 1006, 1012, 1012, 1015,
+ 1015, 1017, 1018, 1021, 1071, 1120, 1120, 1122, 1122, 1124, 1124, 1126, 1126, 1128, 1128, 1130,
+ 1130, 1132, 1132, 1134, 1134, 1136, 1136, 1138, 1138, 1140, 1140, 1142, 1142, 1144, 1144, 1146,
+ 1146, 1148, 1148, 1150, 1150, 1152, 1152, 1162, 1162, 1164, 1164, 1166, 1166, 1168, 1168, 1170,
+ 1170, 1172, 1172, 1174, 1174, 1176, 1176, 1178, 1178, 1180, 1180, 1182, 1182, 1184, 1184, 1186,
+ 1186, 1188, 1188, 1190, 1190, 1192, 1192, 1194, 1194, 1196, 1196, 1198, 1198, 1200, 1200, 1202,
+ 1202, 1204, 1204, 1206, 1206, 1208, 1208, 1210, 1210, 1212, 1212, 1214, 1214, 1216, 1217, 1219,
+ 1219, 1221, 1221, 1223, 1223, 1225, 1225, 1227, 1227, 1229, 1229, 1232, 1232, 1234, 1234, 1236,
+ 1236, 1238, 1238, 1240, 1240, 1242, 1242, 1244, 1244, 1246, 1246, 1248, 1248, 1250, 1250, 1252,
+ 1252, 1254, 1254, 1256, 1256, 1258, 1258, 1260, 1260, 1262, 1262, 1264, 1264, 1266, 1266, 1268,
+ 1268, 1270, 1270, 1272, 1272, 1274, 1274, 1276, 1276, 1278, 1278, 1280, 1280, 1282, 1282, 1284,
+ 1284, 1286, 1286, 1288, 1288, 1290, 1290, 1292, 1292, 1294, 1294, 1296, 1296, 1298, 1298, 1300,
+ 1300, 1302, 1302, 1304, 1304, 1306, 1306, 1308, 1308, 1310, 1310, 1312, 1312, 1314, 1314, 1316,
+ 1316, 1318, 1318, 1320, 1320, 1322, 1322, 1324, 1324, 1326, 1326, 1329, 1366, 4256, 4293, 4295,
+ 4295, 4301, 4301, 5024, 5109, 7312, 7354, 7357, 7359, 7680, 7680, 7682, 7682, 7684, 7684, 7686,
+ 7686, 7688, 7688, 7690, 7690, 7692, 7692, 7694, 7694, 7696, 7696, 7698, 7698, 7700, 7700, 7702,
+ 7702, 7704, 7704, 7706, 7706, 7708, 7708, 7710, 7710, 7712, 7712, 7714, 7714, 7716, 7716, 7718,
+ 7718, 7720, 7720, 7722, 7722, 7724, 7724, 7726, 7726, 7728, 7728, 7730, 7730, 7732, 7732, 7734,
+ 7734, 7736, 7736, 7738, 7738, 7740, 7740, 7742, 7742, 7744, 7744, 7746, 7746, 7748, 7748, 7750,
+ 7750, 7752, 7752, 7754, 7754, 7756, 7756, 7758, 7758, 7760, 7760, 7762, 7762, 7764, 7764, 7766,
+ 7766, 7768, 7768, 7770, 7770, 7772, 7772, 7774, 7774, 7776, 7776, 7778, 7778, 7780, 7780, 7782,
+ 7782, 7784, 7784, 7786, 7786, 7788, 7788, 7790, 7790, 7792, 7792, 7794, 7794, 7796, 7796, 7798,
+ 7798, 7800, 7800, 7802, 7802, 7804, 7804, 7806, 7806, 7808, 7808, 7810, 7810, 7812, 7812, 7814,
+ 7814, 7816, 7816, 7818, 7818, 7820, 7820, 7822, 7822, 7824, 7824, 7826, 7826, 7828, 7828, 7838,
+ 7838, 7840, 7840, 7842, 7842, 7844, 7844, 7846, 7846, 7848, 7848, 7850, 7850, 7852, 7852, 7854,
+ 7854, 7856, 7856, 7858, 7858, 7860, 7860, 7862, 7862, 7864, 7864, 7866, 7866, 7868, 7868, 7870,
+ 7870, 7872, 7872, 7874, 7874, 7876, 7876, 7878, 7878, 7880, 7880, 7882, 7882, 7884, 7884, 7886,
+ 7886, 7888, 7888, 7890, 7890, 7892, 7892, 7894, 7894, 7896, 7896, 7898, 7898, 7900, 7900, 7902,
+ 7902, 7904, 7904, 7906, 7906, 7908, 7908, 7910, 7910, 7912, 7912, 7914, 7914, 7916, 7916, 7918,
+ 7918, 7920, 7920, 7922, 7922, 7924, 7924, 7926, 7926, 7928, 7928, 7930, 7930, 7932, 7932, 7934,
+ 7934, 7944, 7951, 7960, 7965, 7976, 7983, 7992, 7999, 8008, 8013, 8025, 8025, 8027, 8027, 8029,
+ 8029, 8031, 8031, 8040, 8047, 8120, 8123, 8136, 8139, 8152, 8155, 8168, 8172, 8184, 8187, 8450,
+ 8450, 8455, 8455, 8459, 8461, 8464, 8466, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488,
+ 8488, 8490, 8493, 8496, 8499, 8510, 8511, 8517, 8517, 8544, 8559, 8579, 8579, 9398, 9423,
+ 11264, 11311, 11360, 11360, 11362, 11364, 11367, 11367, 11369, 11369, 11371, 11371, 11373,
+ 11376, 11378, 11378, 11381, 11381, 11390, 11392, 11394, 11394, 11396, 11396, 11398, 11398,
+ 11400, 11400, 11402, 11402, 11404, 11404, 11406, 11406, 11408, 11408, 11410, 11410, 11412,
+ 11412, 11414, 11414, 11416, 11416, 11418, 11418, 11420, 11420, 11422, 11422, 11424, 11424,
+ 11426, 11426, 11428, 11428, 11430, 11430, 11432, 11432, 11434, 11434, 11436, 11436, 11438,
+ 11438, 11440, 11440, 11442, 11442, 11444, 11444, 11446, 11446, 11448, 11448, 11450, 11450,
+ 11452, 11452, 11454, 11454, 11456, 11456, 11458, 11458, 11460, 11460, 11462, 11462, 11464,
+ 11464, 11466, 11466, 11468, 11468, 11470, 11470, 11472, 11472, 11474, 11474, 11476, 11476,
+ 11478, 11478, 11480, 11480, 11482, 11482, 11484, 11484, 11486, 11486, 11488, 11488, 11490,
+ 11490, 11499, 11499, 11501, 11501, 11506, 11506, 42560, 42560, 42562, 42562, 42564, 42564,
+ 42566, 42566, 42568, 42568, 42570, 42570, 42572, 42572, 42574, 42574, 42576, 42576, 42578,
+ 42578, 42580, 42580, 42582, 42582, 42584, 42584, 42586, 42586, 42588, 42588, 42590, 42590,
+ 42592, 42592, 42594, 42594, 42596, 42596, 42598, 42598, 42600, 42600, 42602, 42602, 42604,
+ 42604, 42624, 42624, 42626, 42626, 42628, 42628, 42630, 42630, 42632, 42632, 42634, 42634,
+ 42636, 42636, 42638, 42638, 42640, 42640, 42642, 42642, 42644, 42644, 42646, 42646, 42648,
+ 42648, 42650, 42650, 42786, 42786, 42788, 42788, 42790, 42790, 42792, 42792, 42794, 42794,
+ 42796, 42796, 42798, 42798, 42802, 42802, 42804, 42804, 42806, 42806, 42808, 42808, 42810,
+ 42810, 42812, 42812, 42814, 42814, 42816, 42816, 42818, 42818, 42820, 42820, 42822, 42822,
+ 42824, 42824, 42826, 42826, 42828, 42828, 42830, 42830, 42832, 42832, 42834, 42834, 42836,
+ 42836, 42838, 42838, 42840, 42840, 42842, 42842, 42844, 42844, 42846, 42846, 42848, 42848,
+ 42850, 42850, 42852, 42852, 42854, 42854, 42856, 42856, 42858, 42858, 42860, 42860, 42862,
+ 42862, 42873, 42873, 42875, 42875, 42877, 42878, 42880, 42880, 42882, 42882, 42884, 42884,
+ 42886, 42886, 42891, 42891, 42893, 42893, 42896, 42896, 42898, 42898, 42902, 42902, 42904,
+ 42904, 42906, 42906, 42908, 42908, 42910, 42910, 42912, 42912, 42914, 42914, 42916, 42916,
+ 42918, 42918, 42920, 42920, 42922, 42926, 42928, 42932, 42934, 42934, 42936, 42936, 42938,
+ 42938, 42940, 42940, 42942, 42942, 42944, 42944, 42946, 42946, 42948, 42951, 42953, 42953,
+ 42960, 42960, 42966, 42966, 42968, 42968, 42997, 42997, 65313, 65338, 66560, 66599, 66736,
+ 66771, 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 68736, 68786, 71840, 71871,
+ 93760, 93791, 119808, 119833, 119860, 119885, 119912, 119937, 119964, 119964, 119966, 119967,
+ 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119989, 120016, 120041, 120068, 120069,
+ 120071, 120074, 120077, 120084, 120086, 120092, 120120, 120121, 120123, 120126, 120128, 120132,
+ 120134, 120134, 120138, 120144, 120172, 120197, 120224, 120249, 120276, 120301, 120328, 120353,
+ 120380, 120405, 120432, 120457, 120488, 120512, 120546, 120570, 120604, 120628, 120662, 120686,
+ 120720, 120744, 120778, 120778, 125184, 125217, 127280, 127305, 127312, 127337, 127344, 127369,
+};
+
+inline constexpr uint64_t k_upper_ascii[2] = {0x0, 0x7fffffe};
+
+inline constexpr uint32_t k_lower[] = {
+ 97, 122, 170, 170, 181, 181, 186, 186, 223, 246, 248, 255, 257, 257, 259, 259, 261, 261, 263,
+ 263, 265, 265, 267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, 277, 279, 279, 281, 281,
+ 283, 283, 285, 285, 287, 287, 289, 289, 291, 291, 293, 293, 295, 295, 297, 297, 299, 299, 301,
+ 301, 303, 303, 305, 305, 307, 307, 309, 309, 311, 312, 314, 314, 316, 316, 318, 318, 320, 320,
+ 322, 322, 324, 324, 326, 326, 328, 329, 331, 331, 333, 333, 335, 335, 337, 337, 339, 339, 341,
+ 341, 343, 343, 345, 345, 347, 347, 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, 359,
+ 361, 361, 363, 363, 365, 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, 375, 378, 378, 380,
+ 380, 382, 384, 387, 387, 389, 389, 392, 392, 396, 397, 402, 402, 405, 405, 409, 411, 414, 414,
+ 417, 417, 419, 419, 421, 421, 424, 424, 426, 427, 429, 429, 432, 432, 436, 436, 438, 438, 441,
+ 442, 445, 447, 454, 454, 457, 457, 460, 460, 462, 462, 464, 464, 466, 466, 468, 468, 470, 470,
+ 472, 472, 474, 474, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485, 487, 487, 489, 489, 491,
+ 491, 493, 493, 495, 496, 499, 499, 501, 501, 505, 505, 507, 507, 509, 509, 511, 511, 513, 513,
+ 515, 515, 517, 517, 519, 519, 521, 521, 523, 523, 525, 525, 527, 527, 529, 529, 531, 531, 533,
+ 533, 535, 535, 537, 537, 539, 539, 541, 541, 543, 543, 545, 545, 547, 547, 549, 549, 551, 551,
+ 553, 553, 555, 555, 557, 557, 559, 559, 561, 561, 563, 569, 572, 572, 575, 576, 578, 578, 583,
+ 583, 585, 585, 587, 587, 589, 589, 591, 659, 661, 696, 704, 705, 736, 740, 837, 837, 881, 881,
+ 883, 883, 887, 887, 890, 893, 912, 912, 940, 974, 976, 977, 981, 983, 985, 985, 987, 987, 989,
+ 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, 1003, 1005, 1005,
+ 1007, 1011, 1013, 1013, 1016, 1016, 1019, 1020, 1072, 1119, 1121, 1121, 1123, 1123, 1125, 1125,
+ 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, 1135, 1137, 1137, 1139, 1139, 1141, 1141,
+ 1143, 1143, 1145, 1145, 1147, 1147, 1149, 1149, 1151, 1151, 1153, 1153, 1163, 1163, 1165, 1165,
+ 1167, 1167, 1169, 1169, 1171, 1171, 1173, 1173, 1175, 1175, 1177, 1177, 1179, 1179, 1181, 1181,
+ 1183, 1183, 1185, 1185, 1187, 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197, 1197,
+ 1199, 1199, 1201, 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, 1213,
+ 1215, 1215, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, 1228, 1230, 1231,
+ 1233, 1233, 1235, 1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, 1243, 1245, 1245, 1247, 1247,
+ 1249, 1249, 1251, 1251, 1253, 1253, 1255, 1255, 1257, 1257, 1259, 1259, 1261, 1261, 1263, 1263,
+ 1265, 1265, 1267, 1267, 1269, 1269, 1271, 1271, 1273, 1273, 1275, 1275, 1277, 1277, 1279, 1279,
+ 1281, 1281, 1283, 1283, 1285, 1285, 1287, 1287, 1289, 1289, 1291, 1291, 1293, 1293, 1295, 1295,
+ 1297, 1297, 1299, 1299, 1301, 1301, 1303, 1303, 1305, 1305, 1307, 1307, 1309, 1309, 1311, 1311,
+ 1313, 1313, 1315, 1315, 1317, 1317, 1319, 1319, 1321, 1321, 1323, 1323, 1325, 1325, 1327, 1327,
+ 1376, 1416, 4304, 4346, 4348, 4351, 5112, 5117, 7296, 7304, 7424, 7615, 7681, 7681, 7683, 7683,
+ 7685, 7685, 7687, 7687, 7689, 7689, 7691, 7691, 7693, 7693, 7695, 7695, 7697, 7697, 7699, 7699,
+ 7701, 7701, 7703, 7703, 7705, 7705, 7707, 7707, 7709, 7709, 7711, 7711, 7713, 7713, 7715, 7715,
+ 7717, 7717, 7719, 7719, 7721, 7721, 7723, 7723, 7725, 7725, 7727, 7727, 7729, 7729, 7731, 7731,
+ 7733, 7733, 7735, 7735, 7737, 7737, 7739, 7739, 7741, 7741, 7743, 7743, 7745, 7745, 7747, 7747,
+ 7749, 7749, 7751, 7751, 7753, 7753, 7755, 7755, 7757, 7757, 7759, 7759, 7761, 7761, 7763, 7763,
+ 7765, 7765, 7767, 7767, 7769, 7769, 7771, 7771, 7773, 7773, 7775, 7775, 7777, 7777, 7779, 7779,
+ 7781, 7781, 7783, 7783, 7785, 7785, 7787, 7787, 7789, 7789, 7791, 7791, 7793, 7793, 7795, 7795,
+ 7797, 7797, 7799, 7799, 7801, 7801, 7803, 7803, 7805, 7805, 7807, 7807, 7809, 7809, 7811, 7811,
+ 7813, 7813, 7815, 7815, 7817, 7817, 7819, 7819, 7821, 7821, 7823, 7823, 7825, 7825, 7827, 7827,
+ 7829, 7837, 7839, 7839, 7841, 7841, 7843, 7843, 7845, 7845, 7847, 7847, 7849, 7849, 7851, 7851,
+ 7853, 7853, 7855, 7855, 7857, 7857, 7859, 7859, 7861, 7861, 7863, 7863, 7865, 7865, 7867, 7867,
+ 7869, 7869, 7871, 7871, 7873, 7873, 7875, 7875, 7877, 7877, 7879, 7879, 7881, 7881, 7883, 7883,
+ 7885, 7885, 7887, 7887, 7889, 7889, 7891, 7891, 7893, 7893, 7895, 7895, 7897, 7897, 7899, 7899,
+ 7901, 7901, 7903, 7903, 7905, 7905, 7907, 7907, 7909, 7909, 7911, 7911, 7913, 7913, 7915, 7915,
+ 7917, 7917, 7919, 7919, 7921, 7921, 7923, 7923, 7925, 7925, 7927, 7927, 7929, 7929, 7931, 7931,
+ 7933, 7933, 7935, 7943, 7952, 7957, 7968, 7975, 7984, 7991, 8000, 8005, 8016, 8023, 8032, 8039,
+ 8048, 8061, 8064, 8071, 8080, 8087, 8096, 8103, 8112, 8116, 8118, 8119, 8126, 8126, 8130, 8132,
+ 8134, 8135, 8144, 8147, 8150, 8151, 8160, 8167, 8178, 8180, 8182, 8183, 8305, 8305, 8319, 8319,
+ 8336, 8348, 8458, 8458, 8462, 8463, 8467, 8467, 8495, 8495, 8500, 8500, 8505, 8505, 8508, 8509,
+ 8518, 8521, 8526, 8526, 8560, 8575, 8580, 8580, 9424, 9449, 11312, 11359, 11361, 11361, 11365,
+ 11366, 11368, 11368, 11370, 11370, 11372, 11372, 11377, 11377, 11379, 11380, 11382, 11389,
+ 11393, 11393, 11395, 11395, 11397, 11397, 11399, 11399, 11401, 11401, 11403, 11403, 11405,
+ 11405, 11407, 11407, 11409, 11409, 11411, 11411, 11413, 11413, 11415, 11415, 11417, 11417,
+ 11419, 11419, 11421, 11421, 11423, 11423, 11425, 11425, 11427, 11427, 11429, 11429, 11431,
+ 11431, 11433, 11433, 11435, 11435, 11437, 11437, 11439, 11439, 11441, 11441, 11443, 11443,
+ 11445, 11445, 11447, 11447, 11449, 11449, 11451, 11451, 11453, 11453, 11455, 11455, 11457,
+ 11457, 11459, 11459, 11461, 11461, 11463, 11463, 11465, 11465, 11467, 11467, 11469, 11469,
+ 11471, 11471, 11473, 11473, 11475, 11475, 11477, 11477, 11479, 11479, 11481, 11481, 11483,
+ 11483, 11485, 11485, 11487, 11487, 11489, 11489, 11491, 11492, 11500, 11500, 11502, 11502,
+ 11507, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 42561, 42561, 42563, 42563, 42565,
+ 42565, 42567, 42567, 42569, 42569, 42571, 42571, 42573, 42573, 42575, 42575, 42577, 42577,
+ 42579, 42579, 42581, 42581, 42583, 42583, 42585, 42585, 42587, 42587, 42589, 42589, 42591,
+ 42591, 42593, 42593, 42595, 42595, 42597, 42597, 42599, 42599, 42601, 42601, 42603, 42603,
+ 42605, 42605, 42625, 42625, 42627, 42627, 42629, 42629, 42631, 42631, 42633, 42633, 42635,
+ 42635, 42637, 42637, 42639, 42639, 42641, 42641, 42643, 42643, 42645, 42645, 42647, 42647,
+ 42649, 42649, 42651, 42653, 42787, 42787, 42789, 42789, 42791, 42791, 42793, 42793, 42795,
+ 42795, 42797, 42797, 42799, 42801, 42803, 42803, 42805, 42805, 42807, 42807, 42809, 42809,
+ 42811, 42811, 42813, 42813, 42815, 42815, 42817, 42817, 42819, 42819, 42821, 42821, 42823,
+ 42823, 42825, 42825, 42827, 42827, 42829, 42829, 42831, 42831, 42833, 42833, 42835, 42835,
+ 42837, 42837, 42839, 42839, 42841, 42841, 42843, 42843, 42845, 42845, 42847, 42847, 42849,
+ 42849, 42851, 42851, 42853, 42853, 42855, 42855, 42857, 42857, 42859, 42859, 42861, 42861,
+ 42863, 42872, 42874, 42874, 42876, 42876, 42879, 42879, 42881, 42881, 42883, 42883, 42885,
+ 42885, 42887, 42887, 42892, 42892, 42894, 42894, 42897, 42897, 42899, 42901, 42903, 42903,
+ 42905, 42905, 42907, 42907, 42909, 42909, 42911, 42911, 42913, 42913, 42915, 42915, 42917,
+ 42917, 42919, 42919, 42921, 42921, 42927, 42927, 42933, 42933, 42935, 42935, 42937, 42937,
+ 42939, 42939, 42941, 42941, 42943, 42943, 42945, 42945, 42947, 42947, 42952, 42952, 42954,
+ 42954, 42961, 42961, 42963, 42963, 42965, 42965, 42967, 42967, 42969, 42969, 42994, 42996,
+ 42998, 42998, 43000, 43002, 43824, 43866, 43868, 43881, 43888, 43967, 64256, 64262, 64275,
+ 64279, 65345, 65370, 66600, 66639, 66776, 66811, 66967, 66977, 66979, 66993, 66995, 67001,
+ 67003, 67004, 67456, 67456, 67459, 67461, 67463, 67504, 67506, 67514, 68800, 68850, 71872,
+ 71903, 93792, 93823, 119834, 119859, 119886, 119892, 119894, 119911, 119938, 119963, 119990,
+ 119993, 119995, 119995, 119997, 120003, 120005, 120015, 120042, 120067, 120094, 120119, 120146,
+ 120171, 120198, 120223, 120250, 120275, 120302, 120327, 120354, 120379, 120406, 120431, 120458,
+ 120485, 120514, 120538, 120540, 120545, 120572, 120596, 120598, 120603, 120630, 120654, 120656,
+ 120661, 120688, 120712, 120714, 120719, 120746, 120770, 120772, 120777, 120779, 120779, 122624,
+ 122633, 122635, 122654, 122661, 122666, 122928, 122989, 125218, 125251,
+};
+
+inline constexpr uint64_t k_lower_ascii[2] = {0x0, 0x7fffffe00000000};
+
+inline constexpr uint32_t k_space[] = {
+ 9, 13, 28, 32, 133, 133, 160, 160, 5760, 5760, 8192, 8202, 8232, 8233, 8239, 8239, 8287, 8287,
+ 12288, 12288,
+};
+
+inline constexpr uint64_t k_space_ascii[2] = {0x1f0003e00, 0x0};
+
+inline constexpr uint32_t k_printable[] = {
+ 32, 126, 161, 172, 174, 887, 890, 895, 900, 906, 908, 908, 910, 929, 931, 1327, 1329, 1366,
+ 1369, 1418, 1421, 1423, 1425, 1479, 1488, 1514, 1519, 1524, 1542, 1563, 1565, 1756, 1758, 1805,
+ 1808, 1866, 1869, 1969, 1984, 2042, 2045, 2093, 2096, 2110, 2112, 2139, 2142, 2142, 2144, 2154,
+ 2160, 2190, 2200, 2273, 2275, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482,
+ 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2558,
+ 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617,
+ 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2678,
+ 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757,
+ 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2801, 2809, 2815, 2817, 2819, 2821, 2828,
+ 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2876, 2884, 2887, 2888, 2891, 2893,
+ 2901, 2903, 2908, 2909, 2911, 2915, 2918, 2935, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965,
+ 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016,
+ 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3066, 3072, 3084, 3086, 3088, 3090, 3112, 3114, 3129,
+ 3132, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3162, 3165, 3165, 3168, 3171, 3174, 3183,
+ 3191, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277,
+ 3285, 3286, 3293, 3294, 3296, 3299, 3302, 3311, 3313, 3315, 3328, 3340, 3342, 3344, 3346, 3396,
+ 3398, 3400, 3402, 3407, 3412, 3427, 3430, 3455, 3457, 3459, 3461, 3478, 3482, 3505, 3507, 3515,
+ 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3558, 3567, 3570, 3572,
+ 3585, 3642, 3647, 3675, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3773,
+ 3776, 3780, 3782, 3782, 3784, 3790, 3792, 3801, 3804, 3807, 3840, 3911, 3913, 3948, 3953, 3991,
+ 3993, 4028, 4030, 4044, 4046, 4058, 4096, 4293, 4295, 4295, 4301, 4301, 4304, 4680, 4682, 4685,
+ 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798,
+ 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4988, 4992, 5017,
+ 5024, 5109, 5112, 5117, 5120, 5759, 5761, 5788, 5792, 5880, 5888, 5909, 5919, 5942, 5952, 5971,
+ 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6109, 6112, 6121, 6128, 6137, 6144, 6157, 6159, 6169,
+ 6176, 6264, 6272, 6314, 6320, 6389, 6400, 6430, 6432, 6443, 6448, 6459, 6464, 6464, 6468, 6509,
+ 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6618, 6622, 6683, 6686, 6750, 6752, 6780, 6783, 6793,
+ 6800, 6809, 6816, 6829, 6832, 6862, 6912, 6988, 6992, 7038, 7040, 7155, 7164, 7223, 7227, 7241,
+ 7245, 7304, 7312, 7354, 7357, 7367, 7376, 7418, 7424, 7957, 7960, 7965, 7968, 8005, 8008, 8013,
+ 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8132, 8134, 8147,
+ 8150, 8155, 8157, 8175, 8178, 8180, 8182, 8190, 8208, 8231, 8240, 8286, 8304, 8305, 8308, 8334,
+ 8336, 8348, 8352, 8384, 8400, 8432, 8448, 8587, 8592, 9254, 9280, 9290, 9312, 11123, 11126,
+ 11157, 11159, 11507, 11513, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11632,
+ 11647, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720,
+ 11726, 11728, 11734, 11736, 11742, 11744, 11869, 11904, 11929, 11931, 12019, 12032, 12245,
+ 12272, 12283, 12289, 12351, 12353, 12438, 12441, 12543, 12549, 12591, 12593, 12686, 12688,
+ 12771, 12784, 12830, 12832, 42124, 42128, 42182, 42192, 42539, 42560, 42743, 42752, 42954,
+ 42960, 42961, 42963, 42963, 42965, 42969, 42994, 43052, 43056, 43065, 43072, 43127, 43136,
+ 43205, 43214, 43225, 43232, 43347, 43359, 43388, 43392, 43469, 43471, 43481, 43486, 43518,
+ 43520, 43574, 43584, 43597, 43600, 43609, 43612, 43714, 43739, 43766, 43777, 43782, 43785,
+ 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43883, 43888, 44013, 44016, 44025,
+ 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275,
+ 64279, 64285, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64450,
+ 64467, 64911, 64914, 64967, 64975, 64975, 65008, 65049, 65056, 65106, 65108, 65126, 65128,
+ 65131, 65136, 65140, 65142, 65276, 65281, 65470, 65474, 65479, 65482, 65487, 65490, 65495,
+ 65498, 65500, 65504, 65510, 65512, 65518, 65532, 65533, 65536, 65547, 65549, 65574, 65576,
+ 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65792, 65794, 65799, 65843,
+ 65847, 65934, 65936, 65948, 65952, 65952, 66000, 66045, 66176, 66204, 66208, 66256, 66272,
+ 66299, 66304, 66339, 66349, 66378, 66384, 66426, 66432, 66461, 66463, 66499, 66504, 66517,
+ 66560, 66717, 66720, 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 66927,
+ 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993, 66995, 67001,
+ 67003, 67004, 67072, 67382, 67392, 67413, 67424, 67431, 67456, 67461, 67463, 67504, 67506,
+ 67514, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669,
+ 67671, 67742, 67751, 67759, 67808, 67826, 67828, 67829, 67835, 67867, 67871, 67897, 67903,
+ 67903, 67968, 68023, 68028, 68047, 68050, 68099, 68101, 68102, 68108, 68115, 68117, 68119,
+ 68121, 68149, 68152, 68154, 68159, 68168, 68176, 68184, 68192, 68255, 68288, 68326, 68331,
+ 68342, 68352, 68405, 68409, 68437, 68440, 68466, 68472, 68497, 68505, 68508, 68521, 68527,
+ 68608, 68680, 68736, 68786, 68800, 68850, 68858, 68903, 68912, 68921, 69216, 69246, 69248,
+ 69289, 69291, 69293, 69296, 69297, 69373, 69415, 69424, 69465, 69488, 69513, 69552, 69579,
+ 69600, 69622, 69632, 69709, 69714, 69749, 69759, 69820, 69822, 69826, 69840, 69864, 69872,
+ 69881, 69888, 69940, 69942, 69959, 69968, 70006, 70016, 70111, 70113, 70132, 70144, 70161,
+ 70163, 70209, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70313, 70320,
+ 70378, 70384, 70393, 70400, 70403, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448,
+ 70450, 70451, 70453, 70457, 70459, 70468, 70471, 70472, 70475, 70477, 70480, 70480, 70487,
+ 70487, 70493, 70499, 70502, 70508, 70512, 70516, 70656, 70747, 70749, 70753, 70784, 70855,
+ 70864, 70873, 71040, 71093, 71096, 71133, 71168, 71236, 71248, 71257, 71264, 71276, 71296,
+ 71353, 71360, 71369, 71424, 71450, 71453, 71467, 71472, 71494, 71680, 71739, 71840, 71922,
+ 71935, 71942, 71945, 71945, 71948, 71955, 71957, 71958, 71960, 71989, 71991, 71992, 71995,
+ 72006, 72016, 72025, 72096, 72103, 72106, 72151, 72154, 72164, 72192, 72263, 72272, 72354,
+ 72368, 72440, 72448, 72457, 72704, 72712, 72714, 72758, 72760, 72773, 72784, 72812, 72816,
+ 72847, 72850, 72871, 72873, 72886, 72960, 72966, 72968, 72969, 72971, 73014, 73018, 73018,
+ 73020, 73021, 73023, 73031, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73102, 73104,
+ 73105, 73107, 73112, 73120, 73129, 73440, 73464, 73472, 73488, 73490, 73530, 73534, 73561,
+ 73648, 73648, 73664, 73713, 73727, 74649, 74752, 74862, 74864, 74868, 74880, 75075, 77712,
+ 77810, 77824, 78895, 78912, 78933, 82944, 83526, 92160, 92728, 92736, 92766, 92768, 92777,
+ 92782, 92862, 92864, 92873, 92880, 92909, 92912, 92917, 92928, 92997, 93008, 93017, 93019,
+ 93025, 93027, 93047, 93053, 93071, 93760, 93850, 93952, 94026, 94031, 94087, 94095, 94111,
+ 94176, 94180, 94192, 94193, 94208, 100343, 100352, 101589, 101632, 101640, 110576, 110579,
+ 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933,
+ 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817,
+ 113820, 113823, 118528, 118573, 118576, 118598, 118608, 118723, 118784, 119029, 119040, 119078,
+ 119081, 119154, 119163, 119274, 119296, 119365, 119488, 119507, 119520, 119539, 119552, 119638,
+ 119648, 119672, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974,
+ 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074,
+ 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134,
+ 120138, 120144, 120146, 120485, 120488, 120779, 120782, 121483, 121499, 121503, 121505, 121519,
+ 122624, 122654, 122661, 122666, 122880, 122886, 122888, 122904, 122907, 122913, 122915, 122916,
+ 122918, 122922, 122928, 122989, 123023, 123023, 123136, 123180, 123184, 123197, 123200, 123209,
+ 123214, 123215, 123536, 123566, 123584, 123641, 123647, 123647, 124112, 124153, 124896, 124902,
+ 124904, 124907, 124909, 124910, 124912, 124926, 124928, 125124, 125127, 125142, 125184, 125259,
+ 125264, 125273, 125278, 125279, 126065, 126132, 126209, 126269, 126464, 126467, 126469, 126495,
+ 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521,
+ 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543,
+ 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557,
+ 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583,
+ 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633,
+ 126635, 126651, 126704, 126705, 126976, 127019, 127024, 127123, 127136, 127150, 127153, 127167,
+ 127169, 127183, 127185, 127221, 127232, 127405, 127462, 127490, 127504, 127547, 127552, 127560,
+ 127568, 127569, 127584, 127589, 127744, 128727, 128732, 128748, 128752, 128764, 128768, 128886,
+ 128891, 128985, 128992, 129003, 129008, 129008, 129024, 129035, 129040, 129095, 129104, 129113,
+ 129120, 129159, 129168, 129197, 129200, 129201, 129280, 129619, 129632, 129645, 129648, 129660,
+ 129664, 129672, 129680, 129725, 129727, 129733, 129742, 129755, 129760, 129768, 129776, 129784,
+ 129792, 129938, 129940, 129994, 130032, 130041, 131072, 173791, 173824, 177977, 177984, 178205,
+ 178208, 183969, 183984, 191456, 194560, 195101, 196608, 201546, 201552, 205743, 917760, 917999,
+};
+
+inline constexpr uint64_t k_printable_ascii[2] = {0xffffffff00000000, 0x7fffffffffffffff};
+
+inline constexpr uint32_t k_mark[] = {
+ 768, 879, 1155, 1161, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1552, 1562,
+ 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770, 1773, 1809, 1809, 1840, 1866,
+ 1958, 1968, 2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083, 2085, 2087, 2089, 2093, 2137, 2139,
+ 2200, 2207, 2250, 2273, 2275, 2307, 2362, 2364, 2366, 2383, 2385, 2391, 2402, 2403, 2433, 2435,
+ 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509, 2519, 2519, 2530, 2531, 2558, 2558, 2561, 2563,
+ 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2672, 2673, 2677, 2677, 2689, 2691,
+ 2748, 2748, 2750, 2757, 2759, 2761, 2763, 2765, 2786, 2787, 2810, 2815, 2817, 2819, 2876, 2876,
+ 2878, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2914, 2915, 2946, 2946, 3006, 3010, 3014, 3016,
+ 3018, 3021, 3031, 3031, 3072, 3076, 3132, 3132, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158,
+ 3170, 3171, 3201, 3203, 3260, 3260, 3262, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3298, 3299,
+ 3315, 3315, 3328, 3331, 3387, 3388, 3390, 3396, 3398, 3400, 3402, 3405, 3415, 3415, 3426, 3427,
+ 3457, 3459, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3633, 3633, 3636, 3642,
+ 3655, 3662, 3761, 3761, 3764, 3772, 3784, 3790, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897,
+ 3902, 3903, 3953, 3972, 3974, 3975, 3981, 3991, 3993, 4028, 4038, 4038, 4139, 4158, 4182, 4185,
+ 4190, 4192, 4194, 4196, 4199, 4205, 4209, 4212, 4226, 4237, 4239, 4239, 4250, 4253, 4957, 4959,
+ 5906, 5909, 5938, 5940, 5970, 5971, 6002, 6003, 6068, 6099, 6109, 6109, 6155, 6157, 6159, 6159,
+ 6277, 6278, 6313, 6313, 6432, 6443, 6448, 6459, 6679, 6683, 6741, 6750, 6752, 6780, 6783, 6783,
+ 6832, 6862, 6912, 6916, 6964, 6980, 7019, 7027, 7040, 7042, 7073, 7085, 7142, 7155, 7204, 7223,
+ 7376, 7378, 7380, 7400, 7405, 7405, 7412, 7412, 7415, 7417, 7616, 7679, 8400, 8432, 11503,
+ 11505, 11647, 11647, 11744, 11775, 12330, 12335, 12441, 12442, 42607, 42610, 42612, 42621,
+ 42654, 42655, 42736, 42737, 43010, 43010, 43014, 43014, 43019, 43019, 43043, 43047, 43052,
+ 43052, 43136, 43137, 43188, 43205, 43232, 43249, 43263, 43263, 43302, 43309, 43335, 43347,
+ 43392, 43395, 43443, 43456, 43493, 43493, 43561, 43574, 43587, 43587, 43596, 43597, 43643,
+ 43645, 43696, 43696, 43698, 43700, 43703, 43704, 43710, 43711, 43713, 43713, 43755, 43759,
+ 43765, 43766, 44003, 44010, 44012, 44013, 64286, 64286, 65024, 65039, 65056, 65071, 66045,
+ 66045, 66272, 66272, 66422, 66426, 68097, 68099, 68101, 68102, 68108, 68111, 68152, 68154,
+ 68159, 68159, 68325, 68326, 68900, 68903, 69291, 69292, 69373, 69375, 69446, 69456, 69506,
+ 69509, 69632, 69634, 69688, 69702, 69744, 69744, 69747, 69748, 69759, 69762, 69808, 69818,
+ 69826, 69826, 69888, 69890, 69927, 69940, 69957, 69958, 70003, 70003, 70016, 70018, 70067,
+ 70080, 70089, 70092, 70094, 70095, 70188, 70199, 70206, 70206, 70209, 70209, 70367, 70378,
+ 70400, 70403, 70459, 70460, 70462, 70468, 70471, 70472, 70475, 70477, 70487, 70487, 70498,
+ 70499, 70502, 70508, 70512, 70516, 70709, 70726, 70750, 70750, 70832, 70851, 71087, 71093,
+ 71096, 71104, 71132, 71133, 71216, 71232, 71339, 71351, 71453, 71467, 71724, 71738, 71984,
+ 71989, 71991, 71992, 71995, 71998, 72000, 72000, 72002, 72003, 72145, 72151, 72154, 72160,
+ 72164, 72164, 72193, 72202, 72243, 72249, 72251, 72254, 72263, 72263, 72273, 72283, 72330,
+ 72345, 72751, 72758, 72760, 72767, 72850, 72871, 72873, 72886, 73009, 73014, 73018, 73018,
+ 73020, 73021, 73023, 73029, 73031, 73031, 73098, 73102, 73104, 73105, 73107, 73111, 73459,
+ 73462, 73472, 73473, 73475, 73475, 73524, 73530, 73534, 73538, 78912, 78912, 78919, 78933,
+ 92912, 92916, 92976, 92982, 94031, 94031, 94033, 94087, 94095, 94098, 94180, 94180, 94192,
+ 94193, 113821, 113822, 118528, 118573, 118576, 118598, 119141, 119145, 119149, 119154, 119163,
+ 119170, 119173, 119179, 119210, 119213, 119362, 119364, 121344, 121398, 121403, 121452, 121461,
+ 121461, 121476, 121476, 121499, 121503, 121505, 121519, 122880, 122886, 122888, 122904, 122907,
+ 122913, 122915, 122916, 122918, 122922, 123023, 123023, 123184, 123190, 123566, 123566, 123628,
+ 123631, 124140, 124143, 125136, 125142, 125252, 125258, 917760, 917999,
+};
+
+inline constexpr uint64_t k_mark_ascii[2] = {0x0, 0x0};
+
+inline constexpr uint32_t k_initial[] = {
+ 65, 90, 170, 170, 186, 186, 192, 214, 216, 222, 256, 256, 258, 258, 260, 260, 262, 262, 264,
+ 264, 266, 266, 268, 268, 270, 270, 272, 272, 274, 274, 276, 276, 278, 278, 280, 280, 282, 282,
+ 284, 284, 286, 286, 288, 288, 290, 290, 292, 292, 294, 294, 296, 296, 298, 298, 300, 300, 302,
+ 302, 304, 304, 306, 306, 308, 308, 310, 310, 313, 313, 315, 315, 317, 317, 319, 319, 321, 321,
+ 323, 323, 325, 325, 327, 327, 330, 330, 332, 332, 334, 334, 336, 336, 338, 338, 340, 340, 342,
+ 342, 344, 344, 346, 346, 348, 348, 350, 350, 352, 352, 354, 354, 356, 356, 358, 358, 360, 360,
+ 362, 362, 364, 364, 366, 366, 368, 368, 370, 370, 372, 372, 374, 374, 376, 377, 379, 379, 381,
+ 381, 385, 386, 388, 388, 390, 391, 393, 395, 398, 401, 403, 404, 406, 408, 412, 413, 415, 416,
+ 418, 418, 420, 420, 422, 423, 425, 425, 428, 428, 430, 431, 433, 435, 437, 437, 439, 440, 443,
+ 444, 448, 453, 455, 456, 458, 459, 461, 461, 463, 463, 465, 465, 467, 467, 469, 469, 471, 471,
+ 473, 473, 475, 475, 478, 478, 480, 480, 482, 482, 484, 484, 486, 486, 488, 488, 490, 490, 492,
+ 492, 494, 494, 497, 498, 500, 500, 502, 504, 506, 506, 508, 508, 510, 510, 512, 512, 514, 514,
+ 516, 516, 518, 518, 520, 520, 522, 522, 524, 524, 526, 526, 528, 528, 530, 530, 532, 532, 534,
+ 534, 536, 536, 538, 538, 540, 540, 542, 542, 544, 544, 546, 546, 548, 548, 550, 550, 552, 552,
+ 554, 554, 556, 556, 558, 558, 560, 560, 562, 562, 570, 571, 573, 574, 577, 577, 579, 582, 584,
+ 584, 586, 586, 588, 588, 590, 590, 660, 660, 880, 880, 882, 882, 886, 886, 895, 895, 902, 902,
+ 904, 906, 908, 908, 910, 911, 913, 929, 931, 939, 975, 975, 978, 980, 984, 984, 986, 986, 988,
+ 988, 990, 990, 992, 992, 994, 994, 996, 996, 998, 998, 1000, 1000, 1002, 1002, 1004, 1004,
+ 1006, 1006, 1012, 1012, 1015, 1015, 1017, 1018, 1021, 1071, 1120, 1120, 1122, 1122, 1124, 1124,
+ 1126, 1126, 1128, 1128, 1130, 1130, 1132, 1132, 1134, 1134, 1136, 1136, 1138, 1138, 1140, 1140,
+ 1142, 1142, 1144, 1144, 1146, 1146, 1148, 1148, 1150, 1150, 1152, 1152, 1162, 1162, 1164, 1164,
+ 1166, 1166, 1168, 1168, 1170, 1170, 1172, 1172, 1174, 1174, 1176, 1176, 1178, 1178, 1180, 1180,
+ 1182, 1182, 1184, 1184, 1186, 1186, 1188, 1188, 1190, 1190, 1192, 1192, 1194, 1194, 1196, 1196,
+ 1198, 1198, 1200, 1200, 1202, 1202, 1204, 1204, 1206, 1206, 1208, 1208, 1210, 1210, 1212, 1212,
+ 1214, 1214, 1216, 1217, 1219, 1219, 1221, 1221, 1223, 1223, 1225, 1225, 1227, 1227, 1229, 1229,
+ 1232, 1232, 1234, 1234, 1236, 1236, 1238, 1238, 1240, 1240, 1242, 1242, 1244, 1244, 1246, 1246,
+ 1248, 1248, 1250, 1250, 1252, 1252, 1254, 1254, 1256, 1256, 1258, 1258, 1260, 1260, 1262, 1262,
+ 1264, 1264, 1266, 1266, 1268, 1268, 1270, 1270, 1272, 1272, 1274, 1274, 1276, 1276, 1278, 1278,
+ 1280, 1280, 1282, 1282, 1284, 1284, 1286, 1286, 1288, 1288, 1290, 1290, 1292, 1292, 1294, 1294,
+ 1296, 1296, 1298, 1298, 1300, 1300, 1302, 1302, 1304, 1304, 1306, 1306, 1308, 1308, 1310, 1310,
+ 1312, 1312, 1314, 1314, 1316, 1316, 1318, 1318, 1320, 1320, 1322, 1322, 1324, 1324, 1326, 1326,
+ 1329, 1366, 1488, 1514, 1519, 1522, 1568, 1599, 1601, 1610, 1646, 1647, 1649, 1747, 1749, 1749,
+ 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026,
+ 2048, 2069, 2112, 2136, 2144, 2154, 2160, 2183, 2185, 2190, 2208, 2248, 2308, 2361, 2365, 2365,
+ 2384, 2384, 2392, 2401, 2418, 2432, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482,
+ 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2556, 2556, 2565, 2570,
+ 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654,
+ 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749,
+ 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867,
+ 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960,
+ 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024,
+ 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3165, 3165, 3168, 3169,
+ 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3293, 3294,
+ 3296, 3297, 3313, 3314, 3332, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414,
+ 3423, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632,
+ 3634, 3635, 3648, 3653, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760,
+ 3762, 3763, 3773, 3773, 3776, 3780, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980,
+ 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225,
+ 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4352, 4680, 4682, 4685, 4688, 4694, 4696, 4696,
+ 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805,
+ 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5121, 5740, 5743, 5759,
+ 5761, 5786, 5792, 5866, 5873, 5880, 5888, 5905, 5919, 5937, 5952, 5969, 5984, 5996, 5998, 6000,
+ 6016, 6067, 6108, 6108, 6176, 6210, 6212, 6264, 6272, 6276, 6279, 6312, 6314, 6314, 6320, 6389,
+ 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6656, 6678, 6688, 6740, 6917, 6963,
+ 6981, 6988, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7287, 7312, 7354,
+ 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7680, 7680, 7682, 7682, 7684, 7684,
+ 7686, 7686, 7688, 7688, 7690, 7690, 7692, 7692, 7694, 7694, 7696, 7696, 7698, 7698, 7700, 7700,
+ 7702, 7702, 7704, 7704, 7706, 7706, 7708, 7708, 7710, 7710, 7712, 7712, 7714, 7714, 7716, 7716,
+ 7718, 7718, 7720, 7720, 7722, 7722, 7724, 7724, 7726, 7726, 7728, 7728, 7730, 7730, 7732, 7732,
+ 7734, 7734, 7736, 7736, 7738, 7738, 7740, 7740, 7742, 7742, 7744, 7744, 7746, 7746, 7748, 7748,
+ 7750, 7750, 7752, 7752, 7754, 7754, 7756, 7756, 7758, 7758, 7760, 7760, 7762, 7762, 7764, 7764,
+ 7766, 7766, 7768, 7768, 7770, 7770, 7772, 7772, 7774, 7774, 7776, 7776, 7778, 7778, 7780, 7780,
+ 7782, 7782, 7784, 7784, 7786, 7786, 7788, 7788, 7790, 7790, 7792, 7792, 7794, 7794, 7796, 7796,
+ 7798, 7798, 7800, 7800, 7802, 7802, 7804, 7804, 7806, 7806, 7808, 7808, 7810, 7810, 7812, 7812,
+ 7814, 7814, 7816, 7816, 7818, 7818, 7820, 7820, 7822, 7822, 7824, 7824, 7826, 7826, 7828, 7828,
+ 7838, 7838, 7840, 7840, 7842, 7842, 7844, 7844, 7846, 7846, 7848, 7848, 7850, 7850, 7852, 7852,
+ 7854, 7854, 7856, 7856, 7858, 7858, 7860, 7860, 7862, 7862, 7864, 7864, 7866, 7866, 7868, 7868,
+ 7870, 7870, 7872, 7872, 7874, 7874, 7876, 7876, 7878, 7878, 7880, 7880, 7882, 7882, 7884, 7884,
+ 7886, 7886, 7888, 7888, 7890, 7890, 7892, 7892, 7894, 7894, 7896, 7896, 7898, 7898, 7900, 7900,
+ 7902, 7902, 7904, 7904, 7906, 7906, 7908, 7908, 7910, 7910, 7912, 7912, 7914, 7914, 7916, 7916,
+ 7918, 7918, 7920, 7920, 7922, 7922, 7924, 7924, 7926, 7926, 7928, 7928, 7930, 7930, 7932, 7932,
+ 7934, 7934, 7944, 7951, 7960, 7965, 7976, 7983, 7992, 7999, 8008, 8013, 8025, 8025, 8027, 8027,
+ 8029, 8029, 8031, 8031, 8040, 8047, 8072, 8079, 8088, 8095, 8104, 8111, 8120, 8124, 8136, 8140,
+ 8152, 8155, 8168, 8172, 8184, 8188, 8450, 8450, 8455, 8455, 8459, 8461, 8464, 8466, 8469, 8469,
+ 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8496, 8499, 8501, 8504, 8510, 8511,
+ 8517, 8517, 8579, 8579, 11264, 11311, 11360, 11360, 11362, 11364, 11367, 11367, 11369, 11369,
+ 11371, 11371, 11373, 11376, 11378, 11378, 11381, 11381, 11390, 11392, 11394, 11394, 11396,
+ 11396, 11398, 11398, 11400, 11400, 11402, 11402, 11404, 11404, 11406, 11406, 11408, 11408,
+ 11410, 11410, 11412, 11412, 11414, 11414, 11416, 11416, 11418, 11418, 11420, 11420, 11422,
+ 11422, 11424, 11424, 11426, 11426, 11428, 11428, 11430, 11430, 11432, 11432, 11434, 11434,
+ 11436, 11436, 11438, 11438, 11440, 11440, 11442, 11442, 11444, 11444, 11446, 11446, 11448,
+ 11448, 11450, 11450, 11452, 11452, 11454, 11454, 11456, 11456, 11458, 11458, 11460, 11460,
+ 11462, 11462, 11464, 11464, 11466, 11466, 11468, 11468, 11470, 11470, 11472, 11472, 11474,
+ 11474, 11476, 11476, 11478, 11478, 11480, 11480, 11482, 11482, 11484, 11484, 11486, 11486,
+ 11488, 11488, 11490, 11490, 11499, 11499, 11501, 11501, 11506, 11506, 11568, 11623, 11648,
+ 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726,
+ 11728, 11734, 11736, 11742, 12294, 12294, 12348, 12348, 12353, 12438, 12447, 12447, 12449,
+ 12538, 12543, 12543, 12549, 12591, 12593, 12686, 12704, 12735, 12784, 12799, 13312, 19903,
+ 19968, 40980, 40982, 42124, 42192, 42231, 42240, 42507, 42512, 42527, 42538, 42539, 42560,
+ 42560, 42562, 42562, 42564, 42564, 42566, 42566, 42568, 42568, 42570, 42570, 42572, 42572,
+ 42574, 42574, 42576, 42576, 42578, 42578, 42580, 42580, 42582, 42582, 42584, 42584, 42586,
+ 42586, 42588, 42588, 42590, 42590, 42592, 42592, 42594, 42594, 42596, 42596, 42598, 42598,
+ 42600, 42600, 42602, 42602, 42604, 42604, 42606, 42606, 42624, 42624, 42626, 42626, 42628,
+ 42628, 42630, 42630, 42632, 42632, 42634, 42634, 42636, 42636, 42638, 42638, 42640, 42640,
+ 42642, 42642, 42644, 42644, 42646, 42646, 42648, 42648, 42650, 42650, 42656, 42725, 42786,
+ 42786, 42788, 42788, 42790, 42790, 42792, 42792, 42794, 42794, 42796, 42796, 42798, 42798,
+ 42802, 42802, 42804, 42804, 42806, 42806, 42808, 42808, 42810, 42810, 42812, 42812, 42814,
+ 42814, 42816, 42816, 42818, 42818, 42820, 42820, 42822, 42822, 42824, 42824, 42826, 42826,
+ 42828, 42828, 42830, 42830, 42832, 42832, 42834, 42834, 42836, 42836, 42838, 42838, 42840,
+ 42840, 42842, 42842, 42844, 42844, 42846, 42846, 42848, 42848, 42850, 42850, 42852, 42852,
+ 42854, 42854, 42856, 42856, 42858, 42858, 42860, 42860, 42862, 42862, 42873, 42873, 42875,
+ 42875, 42877, 42878, 42880, 42880, 42882, 42882, 42884, 42884, 42886, 42886, 42891, 42891,
+ 42893, 42893, 42895, 42896, 42898, 42898, 42902, 42902, 42904, 42904, 42906, 42906, 42908,
+ 42908, 42910, 42910, 42912, 42912, 42914, 42914, 42916, 42916, 42918, 42918, 42920, 42920,
+ 42922, 42926, 42928, 42932, 42934, 42934, 42936, 42936, 42938, 42938, 42940, 42940, 42942,
+ 42942, 42944, 42944, 42946, 42946, 42948, 42951, 42953, 42953, 42960, 42960, 42966, 42966,
+ 42968, 42968, 42997, 42997, 42999, 42999, 43003, 43009, 43011, 43013, 43015, 43018, 43020,
+ 43042, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301,
+ 43312, 43334, 43360, 43388, 43396, 43442, 43488, 43492, 43495, 43503, 43514, 43518, 43520,
+ 43560, 43584, 43586, 43588, 43595, 43616, 43631, 43633, 43638, 43642, 43642, 43646, 43695,
+ 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43740, 43744,
+ 43754, 43762, 43762, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822,
+ 43968, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64285,
+ 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324,
+ 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142,
+ 65276, 65313, 65338, 65382, 65391, 65393, 65437, 65440, 65470, 65474, 65479, 65482, 65487,
+ 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599,
+ 65613, 65616, 65629, 65664, 65786, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66368,
+ 66370, 66377, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66560, 66599, 66640,
+ 66717, 66736, 66771, 66816, 66855, 66864, 66915, 66928, 66938, 66940, 66954, 66956, 66962,
+ 66964, 66965, 67072, 67382, 67392, 67413, 67424, 67431, 67584, 67589, 67592, 67592, 67594,
+ 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826,
+ 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096, 68112,
+ 68115, 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68324,
+ 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68864,
+ 68899, 69248, 69289, 69296, 69297, 69376, 69404, 69415, 69415, 69424, 69445, 69488, 69505,
+ 69552, 69572, 69600, 69622, 69635, 69687, 69745, 69746, 69749, 69749, 69763, 69807, 69840,
+ 69864, 69891, 69926, 69956, 69956, 69959, 69959, 69968, 70002, 70006, 70006, 70019, 70066,
+ 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187, 70207, 70208, 70272,
+ 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412,
+ 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480,
+ 70480, 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70753, 70784, 70831, 70852, 70853,
+ 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352,
+ 71352, 71424, 71450, 71488, 71494, 71680, 71723, 71840, 71871, 71935, 71942, 71945, 71945,
+ 71948, 71955, 71957, 71958, 71960, 71983, 71999, 71999, 72001, 72001, 72096, 72103, 72106,
+ 72144, 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272,
+ 72284, 72329, 72349, 72349, 72368, 72440, 72704, 72712, 72714, 72750, 72768, 72768, 72818,
+ 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, 73063, 73064,
+ 73066, 73097, 73112, 73112, 73440, 73458, 73474, 73474, 73476, 73488, 73490, 73523, 73648,
+ 73648, 73728, 74649, 74880, 75075, 77712, 77808, 77824, 78895, 78913, 78918, 82944, 83526,
+ 92160, 92728, 92736, 92766, 92784, 92862, 92880, 92909, 92928, 92975, 93027, 93047, 93053,
+ 93071, 93760, 93791, 93952, 94026, 94032, 94032, 94208, 100343, 100352, 101589, 101632, 101640,
+ 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951, 110960, 111355,
+ 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 119808, 119833, 119860, 119885,
+ 119912, 119937, 119964, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980,
+ 119982, 119989, 120016, 120041, 120068, 120069, 120071, 120074, 120077, 120084, 120086, 120092,
+ 120120, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120172, 120197,
+ 120224, 120249, 120276, 120301, 120328, 120353, 120380, 120405, 120432, 120457, 120488, 120512,
+ 120546, 120570, 120604, 120628, 120662, 120686, 120720, 120744, 120778, 120778, 122634, 122634,
+ 123136, 123180, 123214, 123214, 123536, 123565, 123584, 123627, 124112, 124138, 124896, 124902,
+ 124904, 124907, 124909, 124910, 124912, 124926, 124928, 125124, 125184, 125217, 126464, 126467,
+ 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519,
+ 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539,
+ 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555,
+ 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578,
+ 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627,
+ 126629, 126633, 126635, 126651, 131072, 173791, 173824, 177977, 177984, 178205, 178208, 183969,
+ 183984, 191456, 194560, 195101, 196608, 201546, 201552, 205743,
+};
+
+inline constexpr uint64_t k_initial_ascii[2] = {0x0, 0x7fffffe};
+
+inline constexpr uint32_t k_title[] = {
+ 453, 453, 456, 456, 459, 459, 498, 498, 8072, 8079, 8088, 8095, 8104, 8111, 8124, 8124, 8140,
+ 8140, 8188, 8188,
+};
+
+inline constexpr uint64_t k_title_ascii[2] = {0x0, 0x0};
+
+inline constexpr uint32_t k_lower_from[] = {
+ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
+ 89, 90, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208,
+ 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, 221, 222, 256, 258, 260, 262, 264, 266,
+ 268, 270, 272, 274, 276, 278, 280, 282, 284, 286, 288, 290, 292, 294, 296, 298, 300, 302, 306,
+ 308, 310, 313, 315, 317, 319, 321, 323, 325, 327, 330, 332, 334, 336, 338, 340, 342, 344, 346,
+ 348, 350, 352, 354, 356, 358, 360, 362, 364, 366, 368, 370, 372, 374, 376, 377, 379, 381, 385,
+ 386, 388, 390, 391, 393, 394, 395, 398, 399, 400, 401, 403, 404, 406, 407, 408, 412, 413, 415,
+ 416, 418, 420, 422, 423, 425, 428, 430, 431, 433, 434, 435, 437, 439, 440, 444, 452, 453, 455,
+ 456, 458, 459, 461, 463, 465, 467, 469, 471, 473, 475, 478, 480, 482, 484, 486, 488, 490, 492,
+ 494, 497, 498, 500, 502, 503, 504, 506, 508, 510, 512, 514, 516, 518, 520, 522, 524, 526, 528,
+ 530, 532, 534, 536, 538, 540, 542, 544, 546, 548, 550, 552, 554, 556, 558, 560, 562, 570, 571,
+ 573, 574, 577, 579, 580, 581, 582, 584, 586, 588, 590, 880, 882, 886, 895, 902, 904, 905, 906,
+ 908, 910, 911, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928,
+ 929, 931, 932, 933, 934, 935, 936, 937, 938, 939, 975, 984, 986, 988, 990, 992, 994, 996, 998,
+ 1000, 1002, 1004, 1006, 1012, 1015, 1017, 1018, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028,
+ 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044,
+ 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060,
+ 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1120, 1122, 1124, 1126, 1128,
+ 1130, 1132, 1134, 1136, 1138, 1140, 1142, 1144, 1146, 1148, 1150, 1152, 1162, 1164, 1166, 1168,
+ 1170, 1172, 1174, 1176, 1178, 1180, 1182, 1184, 1186, 1188, 1190, 1192, 1194, 1196, 1198, 1200,
+ 1202, 1204, 1206, 1208, 1210, 1212, 1214, 1216, 1217, 1219, 1221, 1223, 1225, 1227, 1229, 1232,
+ 1234, 1236, 1238, 1240, 1242, 1244, 1246, 1248, 1250, 1252, 1254, 1256, 1258, 1260, 1262, 1264,
+ 1266, 1268, 1270, 1272, 1274, 1276, 1278, 1280, 1282, 1284, 1286, 1288, 1290, 1292, 1294, 1296,
+ 1298, 1300, 1302, 1304, 1306, 1308, 1310, 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, 1329,
+ 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1345,
+ 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361,
+ 1362, 1363, 1364, 1365, 1366, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266,
+ 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282,
+ 4283, 4284, 4285, 4286, 4287, 4288, 4289, 4290, 4291, 4292, 4293, 4295, 4301, 5024, 5025, 5026,
+ 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041, 5042,
+ 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5054, 5055, 5056, 5057, 5058,
+ 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073, 5074,
+ 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5089, 5090,
+ 5091, 5092, 5093, 5094, 5095, 5096, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106,
+ 5107, 5108, 5109, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, 7322, 7323, 7324,
+ 7325, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339, 7340,
+ 7341, 7342, 7343, 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7357, 7358,
+ 7359, 7680, 7682, 7684, 7686, 7688, 7690, 7692, 7694, 7696, 7698, 7700, 7702, 7704, 7706, 7708,
+ 7710, 7712, 7714, 7716, 7718, 7720, 7722, 7724, 7726, 7728, 7730, 7732, 7734, 7736, 7738, 7740,
+ 7742, 7744, 7746, 7748, 7750, 7752, 7754, 7756, 7758, 7760, 7762, 7764, 7766, 7768, 7770, 7772,
+ 7774, 7776, 7778, 7780, 7782, 7784, 7786, 7788, 7790, 7792, 7794, 7796, 7798, 7800, 7802, 7804,
+ 7806, 7808, 7810, 7812, 7814, 7816, 7818, 7820, 7822, 7824, 7826, 7828, 7838, 7840, 7842, 7844,
+ 7846, 7848, 7850, 7852, 7854, 7856, 7858, 7860, 7862, 7864, 7866, 7868, 7870, 7872, 7874, 7876,
+ 7878, 7880, 7882, 7884, 7886, 7888, 7890, 7892, 7894, 7896, 7898, 7900, 7902, 7904, 7906, 7908,
+ 7910, 7912, 7914, 7916, 7918, 7920, 7922, 7924, 7926, 7928, 7930, 7932, 7934, 7944, 7945, 7946,
+ 7947, 7948, 7949, 7950, 7951, 7960, 7961, 7962, 7963, 7964, 7965, 7976, 7977, 7978, 7979, 7980,
+ 7981, 7982, 7983, 7992, 7993, 7994, 7995, 7996, 7997, 7998, 7999, 8008, 8009, 8010, 8011, 8012,
+ 8013, 8025, 8027, 8029, 8031, 8040, 8041, 8042, 8043, 8044, 8045, 8046, 8047, 8072, 8073, 8074,
+ 8075, 8076, 8077, 8078, 8079, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8104, 8105, 8106,
+ 8107, 8108, 8109, 8110, 8111, 8120, 8121, 8122, 8123, 8124, 8136, 8137, 8138, 8139, 8140, 8152,
+ 8153, 8154, 8155, 8168, 8169, 8170, 8171, 8172, 8184, 8185, 8186, 8187, 8188, 8486, 8490, 8491,
+ 8498, 8544, 8545, 8546, 8547, 8548, 8549, 8550, 8551, 8552, 8553, 8554, 8555, 8556, 8557, 8558,
+ 8559, 8579, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410, 9411,
+ 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, 11264, 11265, 11266,
+ 11267, 11268, 11269, 11270, 11271, 11272, 11273, 11274, 11275, 11276, 11277, 11278, 11279,
+ 11280, 11281, 11282, 11283, 11284, 11285, 11286, 11287, 11288, 11289, 11290, 11291, 11292,
+ 11293, 11294, 11295, 11296, 11297, 11298, 11299, 11300, 11301, 11302, 11303, 11304, 11305,
+ 11306, 11307, 11308, 11309, 11310, 11311, 11360, 11362, 11363, 11364, 11367, 11369, 11371,
+ 11373, 11374, 11375, 11376, 11378, 11381, 11390, 11391, 11392, 11394, 11396, 11398, 11400,
+ 11402, 11404, 11406, 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424, 11426,
+ 11428, 11430, 11432, 11434, 11436, 11438, 11440, 11442, 11444, 11446, 11448, 11450, 11452,
+ 11454, 11456, 11458, 11460, 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476, 11478,
+ 11480, 11482, 11484, 11486, 11488, 11490, 11499, 11501, 11506, 42560, 42562, 42564, 42566,
+ 42568, 42570, 42572, 42574, 42576, 42578, 42580, 42582, 42584, 42586, 42588, 42590, 42592,
+ 42594, 42596, 42598, 42600, 42602, 42604, 42624, 42626, 42628, 42630, 42632, 42634, 42636,
+ 42638, 42640, 42642, 42644, 42646, 42648, 42650, 42786, 42788, 42790, 42792, 42794, 42796,
+ 42798, 42802, 42804, 42806, 42808, 42810, 42812, 42814, 42816, 42818, 42820, 42822, 42824,
+ 42826, 42828, 42830, 42832, 42834, 42836, 42838, 42840, 42842, 42844, 42846, 42848, 42850,
+ 42852, 42854, 42856, 42858, 42860, 42862, 42873, 42875, 42877, 42878, 42880, 42882, 42884,
+ 42886, 42891, 42893, 42896, 42898, 42902, 42904, 42906, 42908, 42910, 42912, 42914, 42916,
+ 42918, 42920, 42922, 42923, 42924, 42925, 42926, 42928, 42929, 42930, 42931, 42932, 42934,
+ 42936, 42938, 42940, 42942, 42944, 42946, 42948, 42949, 42950, 42951, 42953, 42960, 42966,
+ 42968, 42997, 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323,
+ 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, 65334, 65335, 65336,
+ 65337, 65338, 66560, 66561, 66562, 66563, 66564, 66565, 66566, 66567, 66568, 66569, 66570,
+ 66571, 66572, 66573, 66574, 66575, 66576, 66577, 66578, 66579, 66580, 66581, 66582, 66583,
+ 66584, 66585, 66586, 66587, 66588, 66589, 66590, 66591, 66592, 66593, 66594, 66595, 66596,
+ 66597, 66598, 66599, 66736, 66737, 66738, 66739, 66740, 66741, 66742, 66743, 66744, 66745,
+ 66746, 66747, 66748, 66749, 66750, 66751, 66752, 66753, 66754, 66755, 66756, 66757, 66758,
+ 66759, 66760, 66761, 66762, 66763, 66764, 66765, 66766, 66767, 66768, 66769, 66770, 66771,
+ 66928, 66929, 66930, 66931, 66932, 66933, 66934, 66935, 66936, 66937, 66938, 66940, 66941,
+ 66942, 66943, 66944, 66945, 66946, 66947, 66948, 66949, 66950, 66951, 66952, 66953, 66954,
+ 66956, 66957, 66958, 66959, 66960, 66961, 66962, 66964, 66965, 68736, 68737, 68738, 68739,
+ 68740, 68741, 68742, 68743, 68744, 68745, 68746, 68747, 68748, 68749, 68750, 68751, 68752,
+ 68753, 68754, 68755, 68756, 68757, 68758, 68759, 68760, 68761, 68762, 68763, 68764, 68765,
+ 68766, 68767, 68768, 68769, 68770, 68771, 68772, 68773, 68774, 68775, 68776, 68777, 68778,
+ 68779, 68780, 68781, 68782, 68783, 68784, 68785, 68786, 71840, 71841, 71842, 71843, 71844,
+ 71845, 71846, 71847, 71848, 71849, 71850, 71851, 71852, 71853, 71854, 71855, 71856, 71857,
+ 71858, 71859, 71860, 71861, 71862, 71863, 71864, 71865, 71866, 71867, 71868, 71869, 71870,
+ 71871, 93760, 93761, 93762, 93763, 93764, 93765, 93766, 93767, 93768, 93769, 93770, 93771,
+ 93772, 93773, 93774, 93775, 93776, 93777, 93778, 93779, 93780, 93781, 93782, 93783, 93784,
+ 93785, 93786, 93787, 93788, 93789, 93790, 93791, 125184, 125185, 125186, 125187, 125188,
+ 125189, 125190, 125191, 125192, 125193, 125194, 125195, 125196, 125197, 125198, 125199, 125200,
+ 125201, 125202, 125203, 125204, 125205, 125206, 125207, 125208, 125209, 125210, 125211, 125212,
+ 125213, 125214, 125215, 125216, 125217,
+};
+
+inline constexpr uint32_t k_lower_to[] = {
+ 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,
+ 116, 117, 118, 119, 120, 121, 122, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235,
+ 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 248, 249, 250, 251, 252, 253, 254, 257,
+ 259, 261, 263, 265, 267, 269, 271, 273, 275, 277, 279, 281, 283, 285, 287, 289, 291, 293, 295,
+ 297, 299, 301, 303, 307, 309, 311, 314, 316, 318, 320, 322, 324, 326, 328, 331, 333, 335, 337,
+ 339, 341, 343, 345, 347, 349, 351, 353, 355, 357, 359, 361, 363, 365, 367, 369, 371, 373, 375,
+ 255, 378, 380, 382, 595, 387, 389, 596, 392, 598, 599, 396, 477, 601, 603, 402, 608, 611, 617,
+ 616, 409, 623, 626, 629, 417, 419, 421, 640, 424, 643, 429, 648, 432, 650, 651, 436, 438, 658,
+ 441, 445, 454, 454, 457, 457, 460, 460, 462, 464, 466, 468, 470, 472, 474, 476, 479, 481, 483,
+ 485, 487, 489, 491, 493, 495, 499, 499, 501, 405, 447, 505, 507, 509, 511, 513, 515, 517, 519,
+ 521, 523, 525, 527, 529, 531, 533, 535, 537, 539, 541, 543, 414, 547, 549, 551, 553, 555, 557,
+ 559, 561, 563, 11365, 572, 410, 11366, 578, 384, 649, 652, 583, 585, 587, 589, 591, 881, 883,
+ 887, 1011, 940, 941, 942, 943, 972, 973, 974, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954,
+ 955, 956, 957, 958, 959, 960, 961, 963, 964, 965, 966, 967, 968, 969, 970, 971, 983, 985, 987,
+ 989, 991, 993, 995, 997, 999, 1001, 1003, 1005, 1007, 952, 1016, 1010, 1019, 891, 892, 893,
+ 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119,
+ 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087,
+ 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103,
+ 1121, 1123, 1125, 1127, 1129, 1131, 1133, 1135, 1137, 1139, 1141, 1143, 1145, 1147, 1149, 1151,
+ 1153, 1163, 1165, 1167, 1169, 1171, 1173, 1175, 1177, 1179, 1181, 1183, 1185, 1187, 1189, 1191,
+ 1193, 1195, 1197, 1199, 1201, 1203, 1205, 1207, 1209, 1211, 1213, 1215, 1231, 1218, 1220, 1222,
+ 1224, 1226, 1228, 1230, 1233, 1235, 1237, 1239, 1241, 1243, 1245, 1247, 1249, 1251, 1253, 1255,
+ 1257, 1259, 1261, 1263, 1265, 1267, 1269, 1271, 1273, 1275, 1277, 1279, 1281, 1283, 1285, 1287,
+ 1289, 1291, 1293, 1295, 1297, 1299, 1301, 1303, 1305, 1307, 1309, 1311, 1313, 1315, 1317, 1319,
+ 1321, 1323, 1325, 1327, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388,
+ 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404,
+ 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, 1414, 11520, 11521, 11522, 11523, 11524,
+ 11525, 11526, 11527, 11528, 11529, 11530, 11531, 11532, 11533, 11534, 11535, 11536, 11537,
+ 11538, 11539, 11540, 11541, 11542, 11543, 11544, 11545, 11546, 11547, 11548, 11549, 11550,
+ 11551, 11552, 11553, 11554, 11555, 11556, 11557, 11559, 11565, 43888, 43889, 43890, 43891,
+ 43892, 43893, 43894, 43895, 43896, 43897, 43898, 43899, 43900, 43901, 43902, 43903, 43904,
+ 43905, 43906, 43907, 43908, 43909, 43910, 43911, 43912, 43913, 43914, 43915, 43916, 43917,
+ 43918, 43919, 43920, 43921, 43922, 43923, 43924, 43925, 43926, 43927, 43928, 43929, 43930,
+ 43931, 43932, 43933, 43934, 43935, 43936, 43937, 43938, 43939, 43940, 43941, 43942, 43943,
+ 43944, 43945, 43946, 43947, 43948, 43949, 43950, 43951, 43952, 43953, 43954, 43955, 43956,
+ 43957, 43958, 43959, 43960, 43961, 43962, 43963, 43964, 43965, 43966, 43967, 5112, 5113, 5114,
+ 5115, 5116, 5117, 4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316,
+ 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325, 4326, 4327, 4328, 4329, 4330, 4331, 4332,
+ 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4349, 4350,
+ 4351, 7681, 7683, 7685, 7687, 7689, 7691, 7693, 7695, 7697, 7699, 7701, 7703, 7705, 7707, 7709,
+ 7711, 7713, 7715, 7717, 7719, 7721, 7723, 7725, 7727, 7729, 7731, 7733, 7735, 7737, 7739, 7741,
+ 7743, 7745, 7747, 7749, 7751, 7753, 7755, 7757, 7759, 7761, 7763, 7765, 7767, 7769, 7771, 7773,
+ 7775, 7777, 7779, 7781, 7783, 7785, 7787, 7789, 7791, 7793, 7795, 7797, 7799, 7801, 7803, 7805,
+ 7807, 7809, 7811, 7813, 7815, 7817, 7819, 7821, 7823, 7825, 7827, 7829, 223, 7841, 7843, 7845,
+ 7847, 7849, 7851, 7853, 7855, 7857, 7859, 7861, 7863, 7865, 7867, 7869, 7871, 7873, 7875, 7877,
+ 7879, 7881, 7883, 7885, 7887, 7889, 7891, 7893, 7895, 7897, 7899, 7901, 7903, 7905, 7907, 7909,
+ 7911, 7913, 7915, 7917, 7919, 7921, 7923, 7925, 7927, 7929, 7931, 7933, 7935, 7936, 7937, 7938,
+ 7939, 7940, 7941, 7942, 7943, 7952, 7953, 7954, 7955, 7956, 7957, 7968, 7969, 7970, 7971, 7972,
+ 7973, 7974, 7975, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991, 8000, 8001, 8002, 8003, 8004,
+ 8005, 8017, 8019, 8021, 8023, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, 8064, 8065, 8066,
+ 8067, 8068, 8069, 8070, 8071, 8080, 8081, 8082, 8083, 8084, 8085, 8086, 8087, 8096, 8097, 8098,
+ 8099, 8100, 8101, 8102, 8103, 8112, 8113, 8048, 8049, 8115, 8050, 8051, 8052, 8053, 8131, 8144,
+ 8145, 8054, 8055, 8160, 8161, 8058, 8059, 8165, 8056, 8057, 8060, 8061, 8179, 969, 107, 229,
+ 8526, 8560, 8561, 8562, 8563, 8564, 8565, 8566, 8567, 8568, 8569, 8570, 8571, 8572, 8573, 8574,
+ 8575, 8580, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, 9437,
+ 9438, 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9448, 9449, 11312, 11313, 11314,
+ 11315, 11316, 11317, 11318, 11319, 11320, 11321, 11322, 11323, 11324, 11325, 11326, 11327,
+ 11328, 11329, 11330, 11331, 11332, 11333, 11334, 11335, 11336, 11337, 11338, 11339, 11340,
+ 11341, 11342, 11343, 11344, 11345, 11346, 11347, 11348, 11349, 11350, 11351, 11352, 11353,
+ 11354, 11355, 11356, 11357, 11358, 11359, 11361, 619, 7549, 637, 11368, 11370, 11372, 593, 625,
+ 592, 594, 11379, 11382, 575, 576, 11393, 11395, 11397, 11399, 11401, 11403, 11405, 11407,
+ 11409, 11411, 11413, 11415, 11417, 11419, 11421, 11423, 11425, 11427, 11429, 11431, 11433,
+ 11435, 11437, 11439, 11441, 11443, 11445, 11447, 11449, 11451, 11453, 11455, 11457, 11459,
+ 11461, 11463, 11465, 11467, 11469, 11471, 11473, 11475, 11477, 11479, 11481, 11483, 11485,
+ 11487, 11489, 11491, 11500, 11502, 11507, 42561, 42563, 42565, 42567, 42569, 42571, 42573,
+ 42575, 42577, 42579, 42581, 42583, 42585, 42587, 42589, 42591, 42593, 42595, 42597, 42599,
+ 42601, 42603, 42605, 42625, 42627, 42629, 42631, 42633, 42635, 42637, 42639, 42641, 42643,
+ 42645, 42647, 42649, 42651, 42787, 42789, 42791, 42793, 42795, 42797, 42799, 42803, 42805,
+ 42807, 42809, 42811, 42813, 42815, 42817, 42819, 42821, 42823, 42825, 42827, 42829, 42831,
+ 42833, 42835, 42837, 42839, 42841, 42843, 42845, 42847, 42849, 42851, 42853, 42855, 42857,
+ 42859, 42861, 42863, 42874, 42876, 7545, 42879, 42881, 42883, 42885, 42887, 42892, 613, 42897,
+ 42899, 42903, 42905, 42907, 42909, 42911, 42913, 42915, 42917, 42919, 42921, 614, 604, 609,
+ 620, 618, 670, 647, 669, 43859, 42933, 42935, 42937, 42939, 42941, 42943, 42945, 42947, 42900,
+ 642, 7566, 42952, 42954, 42961, 42967, 42969, 42998, 65345, 65346, 65347, 65348, 65349, 65350,
+ 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, 65360, 65361, 65362, 65363,
+ 65364, 65365, 65366, 65367, 65368, 65369, 65370, 66600, 66601, 66602, 66603, 66604, 66605,
+ 66606, 66607, 66608, 66609, 66610, 66611, 66612, 66613, 66614, 66615, 66616, 66617, 66618,
+ 66619, 66620, 66621, 66622, 66623, 66624, 66625, 66626, 66627, 66628, 66629, 66630, 66631,
+ 66632, 66633, 66634, 66635, 66636, 66637, 66638, 66639, 66776, 66777, 66778, 66779, 66780,
+ 66781, 66782, 66783, 66784, 66785, 66786, 66787, 66788, 66789, 66790, 66791, 66792, 66793,
+ 66794, 66795, 66796, 66797, 66798, 66799, 66800, 66801, 66802, 66803, 66804, 66805, 66806,
+ 66807, 66808, 66809, 66810, 66811, 66967, 66968, 66969, 66970, 66971, 66972, 66973, 66974,
+ 66975, 66976, 66977, 66979, 66980, 66981, 66982, 66983, 66984, 66985, 66986, 66987, 66988,
+ 66989, 66990, 66991, 66992, 66993, 66995, 66996, 66997, 66998, 66999, 67000, 67001, 67003,
+ 67004, 68800, 68801, 68802, 68803, 68804, 68805, 68806, 68807, 68808, 68809, 68810, 68811,
+ 68812, 68813, 68814, 68815, 68816, 68817, 68818, 68819, 68820, 68821, 68822, 68823, 68824,
+ 68825, 68826, 68827, 68828, 68829, 68830, 68831, 68832, 68833, 68834, 68835, 68836, 68837,
+ 68838, 68839, 68840, 68841, 68842, 68843, 68844, 68845, 68846, 68847, 68848, 68849, 68850,
+ 71872, 71873, 71874, 71875, 71876, 71877, 71878, 71879, 71880, 71881, 71882, 71883, 71884,
+ 71885, 71886, 71887, 71888, 71889, 71890, 71891, 71892, 71893, 71894, 71895, 71896, 71897,
+ 71898, 71899, 71900, 71901, 71902, 71903, 93792, 93793, 93794, 93795, 93796, 93797, 93798,
+ 93799, 93800, 93801, 93802, 93803, 93804, 93805, 93806, 93807, 93808, 93809, 93810, 93811,
+ 93812, 93813, 93814, 93815, 93816, 93817, 93818, 93819, 93820, 93821, 93822, 93823, 125218,
+ 125219, 125220, 125221, 125222, 125223, 125224, 125225, 125226, 125227, 125228, 125229, 125230,
+ 125231, 125232, 125233, 125234, 125235, 125236, 125237, 125238, 125239, 125240, 125241, 125242,
+ 125243, 125244, 125245, 125246, 125247, 125248, 125249, 125250, 125251,
+};
+
+struct LowerMulti { uint32_t from; uint32_t count; uint32_t to[3]; };
+inline constexpr LowerMulti k_lower_multi[] = {
+ {0x130, 2, {0x69, 0x307, 0x0}},
+};
+
+inline constexpr uint32_t k_cased[] = {
+ 65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 442, 444, 447, 452,
+ 659, 661, 687, 880, 883, 886, 887, 891, 893, 895, 895, 902, 902, 904, 906, 908, 908, 910, 929,
+ 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1376, 1416, 4256, 4293, 4295, 4295, 4301, 4301,
+ 4304, 4346, 4349, 4351, 5024, 5109, 5112, 5117, 7296, 7304, 7312, 7354, 7357, 7359, 7424, 7467,
+ 7531, 7543, 7545, 7578, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025,
+ 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140,
+ 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8450, 8450, 8455, 8455, 8458, 8467,
+ 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8500, 8505, 8505,
+ 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8575, 8579, 8580, 9398, 9449, 11264, 11387, 11390,
+ 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 42560, 42605,
+ 42624, 42651, 42786, 42863, 42865, 42887, 42891, 42894, 42896, 42954, 42960, 42961, 42963,
+ 42963, 42965, 42969, 42997, 42998, 43002, 43002, 43824, 43866, 43872, 43880, 43888, 43967,
+ 64256, 64262, 64275, 64279, 65313, 65338, 65345, 65370, 66560, 66639, 66736, 66771, 66776,
+ 66811, 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 66967, 66977, 66979, 66993,
+ 66995, 67001, 67003, 67004, 68736, 68786, 68800, 68850, 71840, 71903, 93760, 93823, 119808,
+ 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982,
+ 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086,
+ 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146,
+ 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630,
+ 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 122624,
+ 122633, 122635, 122654, 122661, 122666, 125184, 125251, 127280, 127305, 127312, 127337, 127344,
+ 127369,
+};
+
+inline constexpr uint32_t k_case_ignorable[] = {
+ 39, 39, 46, 46, 58, 58, 94, 94, 96, 96, 168, 168, 173, 173, 175, 175, 180, 180, 183, 184, 688,
+ 879, 884, 885, 890, 890, 900, 901, 903, 903, 1155, 1161, 1369, 1369, 1375, 1375, 1425, 1469,
+ 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1524, 1524, 1536, 1541, 1552, 1562, 1564, 1564,
+ 1600, 1600, 1611, 1631, 1648, 1648, 1750, 1757, 1759, 1768, 1770, 1773, 1807, 1807, 1809, 1809,
+ 1840, 1866, 1958, 1968, 2027, 2037, 2042, 2042, 2045, 2045, 2070, 2093, 2137, 2139, 2184, 2184,
+ 2192, 2193, 2200, 2207, 2249, 2306, 2362, 2362, 2364, 2364, 2369, 2376, 2381, 2381, 2385, 2391,
+ 2402, 2403, 2417, 2417, 2433, 2433, 2492, 2492, 2497, 2500, 2509, 2509, 2530, 2531, 2558, 2558,
+ 2561, 2562, 2620, 2620, 2625, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2672, 2673, 2677, 2677,
+ 2689, 2690, 2748, 2748, 2753, 2757, 2759, 2760, 2765, 2765, 2786, 2787, 2810, 2815, 2817, 2817,
+ 2876, 2876, 2879, 2879, 2881, 2884, 2893, 2893, 2901, 2902, 2914, 2915, 2946, 2946, 3008, 3008,
+ 3021, 3021, 3072, 3072, 3076, 3076, 3132, 3132, 3134, 3136, 3142, 3144, 3146, 3149, 3157, 3158,
+ 3170, 3171, 3201, 3201, 3260, 3260, 3263, 3263, 3270, 3270, 3276, 3277, 3298, 3299, 3328, 3329,
+ 3387, 3388, 3393, 3396, 3405, 3405, 3426, 3427, 3457, 3457, 3530, 3530, 3538, 3540, 3542, 3542,
+ 3633, 3633, 3636, 3642, 3654, 3662, 3761, 3761, 3764, 3772, 3782, 3782, 3784, 3790, 3864, 3865,
+ 3893, 3893, 3895, 3895, 3897, 3897, 3953, 3966, 3968, 3972, 3974, 3975, 3981, 3991, 3993, 4028,
+ 4038, 4038, 4141, 4144, 4146, 4151, 4153, 4154, 4157, 4158, 4184, 4185, 4190, 4192, 4209, 4212,
+ 4226, 4226, 4229, 4230, 4237, 4237, 4253, 4253, 4348, 4348, 4957, 4959, 5906, 5908, 5938, 5939,
+ 5970, 5971, 6002, 6003, 6068, 6069, 6071, 6077, 6086, 6086, 6089, 6099, 6103, 6103, 6109, 6109,
+ 6155, 6159, 6211, 6211, 6277, 6278, 6313, 6313, 6432, 6434, 6439, 6440, 6450, 6450, 6457, 6459,
+ 6679, 6680, 6683, 6683, 6742, 6742, 6744, 6750, 6752, 6752, 6754, 6754, 6757, 6764, 6771, 6780,
+ 6783, 6783, 6823, 6823, 6832, 6862, 6912, 6915, 6964, 6964, 6966, 6970, 6972, 6972, 6978, 6978,
+ 7019, 7027, 7040, 7041, 7074, 7077, 7080, 7081, 7083, 7085, 7142, 7142, 7144, 7145, 7149, 7149,
+ 7151, 7153, 7212, 7219, 7222, 7223, 7288, 7293, 7376, 7378, 7380, 7392, 7394, 7400, 7405, 7405,
+ 7412, 7412, 7416, 7417, 7468, 7530, 7544, 7544, 7579, 7679, 8125, 8125, 8127, 8129, 8141, 8143,
+ 8157, 8159, 8173, 8175, 8189, 8190, 8203, 8207, 8216, 8217, 8228, 8228, 8231, 8231, 8234, 8238,
+ 8288, 8292, 8294, 8303, 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8432, 11388, 11389, 11503,
+ 11505, 11631, 11631, 11647, 11647, 11744, 11775, 11823, 11823, 12293, 12293, 12330, 12333,
+ 12337, 12341, 12347, 12347, 12441, 12446, 12540, 12542, 40981, 40981, 42232, 42237, 42508,
+ 42508, 42607, 42610, 42612, 42621, 42623, 42623, 42652, 42655, 42736, 42737, 42752, 42785,
+ 42864, 42864, 42888, 42890, 42994, 42996, 43000, 43001, 43010, 43010, 43014, 43014, 43019,
+ 43019, 43045, 43046, 43052, 43052, 43204, 43205, 43232, 43249, 43263, 43263, 43302, 43309,
+ 43335, 43345, 43392, 43394, 43443, 43443, 43446, 43449, 43452, 43453, 43471, 43471, 43493,
+ 43494, 43561, 43566, 43569, 43570, 43573, 43574, 43587, 43587, 43596, 43596, 43632, 43632,
+ 43644, 43644, 43696, 43696, 43698, 43700, 43703, 43704, 43710, 43711, 43713, 43713, 43741,
+ 43741, 43756, 43757, 43763, 43764, 43766, 43766, 43867, 43871, 43881, 43883, 44005, 44005,
+ 44008, 44008, 44013, 44013, 64286, 64286, 64434, 64450, 65024, 65039, 65043, 65043, 65056,
+ 65071, 65106, 65106, 65109, 65109, 65279, 65279, 65287, 65287, 65294, 65294, 65306, 65306,
+ 65342, 65342, 65344, 65344, 65392, 65392, 65438, 65439, 65507, 65507, 65529, 65531, 66045,
+ 66045, 66272, 66272, 66422, 66426, 67456, 67461, 67463, 67504, 67506, 67514, 68097, 68099,
+ 68101, 68102, 68108, 68111, 68152, 68154, 68159, 68159, 68325, 68326, 68900, 68903, 69291,
+ 69292, 69373, 69375, 69446, 69456, 69506, 69509, 69633, 69633, 69688, 69702, 69744, 69744,
+ 69747, 69748, 69759, 69761, 69811, 69814, 69817, 69818, 69821, 69821, 69826, 69826, 69837,
+ 69837, 69888, 69890, 69927, 69931, 69933, 69940, 70003, 70003, 70016, 70017, 70070, 70078,
+ 70089, 70092, 70095, 70095, 70191, 70193, 70196, 70196, 70198, 70199, 70206, 70206, 70209,
+ 70209, 70367, 70367, 70371, 70378, 70400, 70401, 70459, 70460, 70464, 70464, 70502, 70508,
+ 70512, 70516, 70712, 70719, 70722, 70724, 70726, 70726, 70750, 70750, 70835, 70840, 70842,
+ 70842, 70847, 70848, 70850, 70851, 71090, 71093, 71100, 71101, 71103, 71104, 71132, 71133,
+ 71219, 71226, 71229, 71229, 71231, 71232, 71339, 71339, 71341, 71341, 71344, 71349, 71351,
+ 71351, 71453, 71455, 71458, 71461, 71463, 71467, 71727, 71735, 71737, 71738, 71995, 71996,
+ 71998, 71998, 72003, 72003, 72148, 72151, 72154, 72155, 72160, 72160, 72193, 72202, 72243,
+ 72248, 72251, 72254, 72263, 72263, 72273, 72278, 72281, 72283, 72330, 72342, 72344, 72345,
+ 72752, 72758, 72760, 72765, 72767, 72767, 72850, 72871, 72874, 72880, 72882, 72883, 72885,
+ 72886, 73009, 73014, 73018, 73018, 73020, 73021, 73023, 73029, 73031, 73031, 73104, 73105,
+ 73109, 73109, 73111, 73111, 73459, 73460, 73472, 73473, 73526, 73530, 73536, 73536, 73538,
+ 73538, 78896, 78912, 78919, 78933, 92912, 92916, 92976, 92982, 92992, 92995, 94031, 94031,
+ 94095, 94111, 94176, 94177, 94179, 94180, 110576, 110579, 110581, 110587, 110589, 110590,
+ 113821, 113822, 113824, 113827, 118528, 118573, 118576, 118598, 119143, 119145, 119155, 119170,
+ 119173, 119179, 119210, 119213, 119362, 119364, 121344, 121398, 121403, 121452, 121461, 121461,
+ 121476, 121476, 121499, 121503, 121505, 121519, 122880, 122886, 122888, 122904, 122907, 122913,
+ 122915, 122916, 122918, 122922, 122928, 122989, 123023, 123023, 123184, 123197, 123566, 123566,
+ 123628, 123631, 124139, 124143, 125136, 125142, 125252, 125259, 127995, 127999, 917505, 917505,
+ 917536, 917631, 917760, 917999,
+};
diff --git a/laya-cpp/tests/foundation_test.cpp b/laya-cpp/tests/foundation_test.cpp
new file mode 100644
index 00000000..2170f078
--- /dev/null
+++ b/laya-cpp/tests/foundation_test.cpp
@@ -0,0 +1,222 @@
+#include <cstring>
+
+#include "golden.hpp"
+#include "laya/pyjson.hpp"
+#include "laya/pyvalue.hpp"
+#include "laya/sha256.hpp"
+#include "laya/unicodedata.hpp"
+
+using laya_test::At;
+using laya_test::Value;
+namespace ud = laya::unicodedata;
+namespace pv = laya::pyvalue;
+
+// ---- sha256: FIPS 180-4 / NIST vectors and Python hashlib goldens --------------------------------
+TEST(Sha256, FipsAndHashlibVectors) {
+ EXPECT_EQ(laya::Sha256Hex(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+ EXPECT_EQ(laya::Sha256Hex("abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
+ EXPECT_EQ(laya::Sha256Hex("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"),
+ "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
+ EXPECT_EQ(laya::Sha256Hex(std::string(1000000, 'a')),
+ "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0");
+ EXPECT_EQ(laya::Sha256Hex(std::string(55, 'a')), // padding edge: 55, 56, 63, 64 byte messages
+ "9f4390f8d30c2dd92ec9f095b65e2b9ae9b0a925a5258e241c9f1e910f734318");
+ EXPECT_EQ(laya::Sha256Hex(std::string(56, 'a')),
+ "b35439a4ac6f0948b6d6f9e3c6af0f5f590ce20f1bde7090ef7970686ec6738a");
+ EXPECT_EQ(laya::Sha256Hex(std::string(63, 'a')),
+ "7d3e74a05d7db15bce4ad9ec0658ea98e3f06eeecf16b4c6fff2da457ddc2f34");
+ EXPECT_EQ(laya::Sha256Hex(std::string(64, 'a')),
+ "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb");
+}
+
+TEST(Sha256, IncrementalEqualsOneShot) {
+ std::string data(777, 'x');
+ for (size_t i = 0; i < data.size(); i++) data[i] = char(i * 31);
+ laya::Sha256 h;
+ for (size_t i = 0; i < data.size(); i += 13) h.Update(data.data() + i, std::min<size_t>(13, data.size() - i));
+ EXPECT_EQ(laya::ToHex(h.Finish()), laya::Sha256Hex(data));
+}
+
+// ---- unicodedata: whole-Unicode digests against CPython ------------------------------------------
+namespace {
+std::string PredicateDigest(bool (*pred)(char32_t)) {
+ std::string bits;
+ for (char32_t r = 0; r < 0x110000; r++) bits.push_back(pred(r) ? '1' : '0');
+ return laya::Sha256Hex(bits);
+}
+} // namespace
+
+TEST(Unicode, VersionAndPredicatesMatchCPythonEverywhere) {
+ LAYA_GOLDEN_LIST(d, "lang_detect.json", "unicode_digests");
+ EXPECT_STREQ(ud::kVersion, At(laya_test::Load("lang_detect.json", true), {"unicode_version"})->as_string().c_str());
+ struct P { const char* name; bool (*fn)(char32_t); };
+ const P preds[] = {{"alpha", ud::IsAlpha}, {"word", ud::IsWord}, {"combining", ud::IsCombining},
+ {"digit", ud::IsDigit}, {"upper", ud::IsUpper}, {"lower", ud::IsLower},
+ {"space", ud::IsSpace}, {"mark", ud::IsMark}, {"initial", ud::IsInitial}};
+ for (const P& p : preds) {
+ ASSERT_NE(d->Find(p.name), nullptr) << p.name;
+ EXPECT_EQ(PredicateDigest(p.fn), d->Find(p.name)->as_string()) << p.name;
+ }
+}
+
+TEST(Unicode, LowerMatchesCPythonEverywhere) {
+ LAYA_GOLDEN_LIST(d, "lang_detect.json", "unicode_digests", "python_lower");
+ laya::Sha256 h;
+ for (char32_t r = 0; r < 0x110000; r++) {
+ if (r >= 0xD800 && r <= 0xDFFF) continue;
+ std::string one;
+ ud::AppendRune(one, r);
+ std::string low = ud::Lower(one), line = std::to_string(r) + ":";
+ for (size_t i = 0; i < low.size();) {
+ ud::Decoded x = ud::DecodeRune(std::string_view(low).substr(i));
+ if (i > 0) line += ",";
+ line += std::to_string(x.rune);
+ i += x.width;
+ }
+ line += ";";
+ h.Update(line);
+ }
+ EXPECT_EQ(laya::ToHex(h.Finish()), d->as_string());
+}
+
+TEST(Unicode, FinalSigmaAndPythonLowerRows) {
+ LAYA_GOLDEN_LIST(rows, "go_extra.json", "python_lower", "cases");
+ for (size_t i = 0; i < rows->size(); i++) {
+ const Value& c = rows->value(i);
+ EXPECT_EQ(ud::Lower(c.Find("text")->as_string()), c.Find("lower")->as_string()) << "row " << i;
+ }
+}
+
+TEST(Unicode, InvalidUtf8MatchesPython) {
+ LAYA_GOLDEN_LIST(fam, "go_extra.json", "invalid_utf8");
+ const Value& hex = *fam->Find("hex");
+ const Value& decoded = *fam->Find("decoded");
+ ASSERT_EQ(hex.size(), 3000u);
+ for (size_t i = 0; i < hex.size(); i++) {
+ const std::string& h = hex.value(i).as_string();
+ std::string bytes;
+ for (size_t k = 0; k + 1 < h.size(); k += 2) bytes.push_back(char(std::stoi(h.substr(k, 2), nullptr, 16)));
+ EXPECT_EQ(ud::Lossy(bytes), decoded.value(i).as_string()) << "row " << i << " " << h;
+ }
+}
+
+TEST(Unicode, StripFieldsDecimal) {
+ EXPECT_EQ(ud::Strip(" a b\t\xe3\x80\x80"), "a b");
+ EXPECT_EQ(ud::StripNumeric("\x1f 12 "), "\x1f 12");
+ EXPECT_EQ(ud::Fields(" a b\n").size(), 2u);
+ EXPECT_EQ(ud::DecimalASCII("\xd9\xa1\xd9\xa2"), "12");
+ EXPECT_TRUE(ud::IsBlank(" \t"));
+ EXPECT_TRUE(ud::IsUpperString("AB1"));
+ EXPECT_FALSE(ud::IsUpperString("Ab"));
+}
+
+// ---- pyjson: float repr, str repr, percent, round4 -----------------------------------------------
+class DoublesChunk : public ::testing::TestWithParam<int> {};
+TEST_P(DoublesChunk, ReprFloatMatchesCPython) {
+ LAYA_GOLDEN_LIST(rows, "python_json.json", "doubles");
+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < std::min(rows->size(), (size_t(GetParam()) + 1) * laya_test::kChunk); i++) {
+ uint64_t bits = laya_test::BitsOf(*rows->value(i).Find("bits"));
+ double v;
+ std::memcpy(&v, &bits, 8);
+ EXPECT_EQ(laya::pyjson::ReprFloat(v), rows->value(i).Find("repr")->as_string()) << "bits " << bits;
+ }
+}
+INSTANTIATE_TEST_SUITE_P(Golden, DoublesChunk, ::testing::ValuesIn(laya_test::Chunks("python_json.json", {"doubles"})));
+
+TEST(PyJson, ReprStringsDigestPercentRound4) {
+ LAYA_GOLDEN_LIST(rs, "python_json.json", "repr_strings");
+ for (size_t i = 0; i < rs->size(); i++) {
+ EXPECT_EQ(laya::pyjson::Repr(rs->value(i).value(0).as_string()), rs->value(i).value(1).as_string());
+ }
+ laya::Sha256 h; // Repr("a"+chr(r)+"b") over every non-surrogate code point
+ for (char32_t r = 0; r < 0x110000; r++) {
+ if (r >= 0xD800 && r <= 0xDFFF) continue;
+ std::string s = "a";
+ ud::AppendRune(s, r);
+ s += "b";
+ h.Update(laya::pyjson::Repr(s));
+ }
+ EXPECT_EQ(laya::ToHex(h.Finish()), At(laya_test::Load("python_json.json", true), {"repr_digest"})->as_string());
+ const Value* pc = At(laya_test::Load("python_json.json", true), {"percent0"});
+ for (size_t i = 0; i < pc->size(); i++) {
+ EXPECT_EQ(laya::pyjson::Percent0(laya_test::Num(pc->value(i).value(0))), pc->value(i).value(1).as_string());
+ }
+ const Value* groups = At(laya_test::Load("python_json.json", true), {"round4"});
+ ASSERT_NE(groups->Find("halfway"), nullptr);
+ for (size_t g = 0; g < groups->size(); g++) {
+ const Value& rows = groups->value(g);
+ for (size_t i = 0; i < rows.size(); i++) {
+ uint64_t bits = laya_test::BitsOf(*rows.value(i).Find("bits"));
+ double v;
+ std::memcpy(&v, &bits, 8);
+ EXPECT_EQ(laya::pyjson::Round4(v), laya_test::Num(*rows.value(i).Find("r4"))) << groups->key(g) << " " << v;
+ }
+ }
+}
+
+class Round4Chunk : public ::testing::TestWithParam<int> {};
+TEST_P(Round4Chunk, HalfEvenMatchesPython) {
+ LAYA_GOLDEN_LIST(vals, "go_extra.json", "round4", "values");
+ const Value* want = At(laya_test::Load("go_extra.json", true), {"round4", "rounded"});
+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < std::min(vals->size(), (size_t(GetParam()) + 1) * laya_test::kChunk); i++) {
+ double got = laya::pyjson::Round4(laya_test::Num(vals->value(i))), w = laya_test::Num(want->value(i));
+ EXPECT_EQ(std::memcmp(&got, &w, 8), 0) << "row " << i << " " << got << " vs " << w;
+ }
+}
+INSTANTIATE_TEST_SUITE_P(Golden, Round4Chunk, ::testing::ValuesIn(laya_test::Chunks("go_extra.json", {"round4", "values"})));
+
+// ---- pyvalue: json.loads / json.dumps ------------------------------------------------------------
+class StatesChunk : public ::testing::TestWithParam<int> {};
+TEST_P(StatesChunk, ParseAndDumpsMatchPython) {
+ LAYA_GOLDEN_LIST(cases, "go_extra.json", "states", "cases");
+ for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < std::min(cases->size(), (size_t(GetParam()) + 1) * laya_test::kChunk); i++) {
+ const std::string& text = cases->value(i).Find("text")->as_string();
+ auto v = pv::Parse(text);
+ ASSERT_TRUE(v.ok()) << i << " " << v.error();
+ std::string got;
+ if (v.value().kind() == pv::Value::Kind::String) {
+ got = v.value().as_string();
+ } else {
+ auto d = pv::Dumps(v.value());
+ ASSERT_TRUE(d.ok()) << i << " " << d.error();
+ got = d.value();
+ }
+ EXPECT_EQ(got, cases->value(i).Find("serialized")->as_string()) << "row " << i;
+ EXPECT_EQ(v.value().kind() == pv::Value::Kind::Array, cases->value(i).Find("list")->as_bool()) << i;
+ }
+}
+INSTANTIATE_TEST_SUITE_P(Golden, StatesChunk, ::testing::ValuesIn(laya_test::Chunks("go_extra.json", {"states", "cases"})));
+
+TEST(PyValue, PythonRejectsAndLoneSurrogates) {
+ LAYA_GOLDEN_LIST(bad, "go_extra.json", "states", "python_rejects");
+ for (size_t i = 0; i < bad->size(); i++) EXPECT_FALSE(pv::Parse(bad->value(i).as_string()).ok()) << bad->value(i).as_string().substr(0, 40);
+ const Value* lone = At(laya_test::Load("go_extra.json", true), {"states", "go_refuses"});
+ for (size_t i = 0; i < lone->size(); i++) {
+ auto r = pv::Parse(lone->value(i).as_string());
+ ASSERT_FALSE(r.ok());
+ EXPECT_NE(r.error().find("lone surrogate"), std::string::npos) << r.error();
+ }
+}
+
+TEST(PyValue, DuplicateKeysNaNDepthBigIntRepr) {
+ auto d = pv::Parse("{\"a\": 1, \"b\": 2, \"a\": 3}");
+ ASSERT_TRUE(d.ok());
+ EXPECT_EQ(pv::Dumps(d.value()).value(), "{\"a\": 3, \"b\": 2}"); // first position, last value
+ auto n = pv::Parse("[NaN, Infinity, -Infinity, 2, 2.0, 1e400, -0, 12345678901234567890123]");
+ ASSERT_TRUE(n.ok());
+ EXPECT_EQ(pv::Dumps(n.value()).value(), "[NaN, Infinity, -Infinity, 2, 2.0, Infinity, 0, 12345678901234567890123]");
+ EXPECT_EQ(pv::Repr(n.value()), "[nan, inf, -inf, 2, 2.0, inf, 0, 12345678901234567890123]");
+ std::string deep(600, '[');
+ deep += std::string(600, ']');
+ EXPECT_FALSE(pv::Parse(deep).ok());
+ std::string ok(pv::kMaxDepth, '[');
+ ok += std::string(pv::kMaxDepth, ']');
+ EXPECT_TRUE(pv::Parse(ok).ok());
+ EXPECT_FALSE(pv::Parse(std::string(4301, '9')).ok());
+ EXPECT_TRUE(pv::Parse(std::string(4300, '9')).ok());
+ EXPECT_FALSE(pv::Parse("\"\\ud800\"").ok());
+ EXPECT_FALSE(pv::Parse("\xff").ok());
+ auto o = pv::Parse("{\"k\": [\"it's\", null, true]}");
+ EXPECT_EQ(pv::Repr(o.value()), "{'k': [\"it's\", None, True]}");
+ EXPECT_EQ(pv::TypeName(o.value()), "dict");
+}
diff --git a/laya-cpp/tests/golden.hpp b/laya-cpp/tests/golden.hpp
new file mode 100644
index 00000000..c0226d12
--- /dev/null
+++ b/laya-cpp/tests/golden.hpp
@@ -0,0 +1,87 @@
+#pragma once
+// Golden loading for the parity tests. The goldens are regenerated from the Python reference at
+// the commit under test (CI) or by the lane script (local) into $LAYA_GOLDENS; none is committed.
+// With LAYA_REQUIRE_GOLDENS=1 (CI) a missing golden is a test FAILURE, never a skip.
+#include <gtest/gtest.h>
+
+#include <cstdlib>
+#include <fstream>
+#include <map>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "laya/pyvalue.hpp"
+
+namespace laya_test {
+
+using laya::pyvalue::Value;
+
+// The parsed golden file, or nullptr. report=false is for instantiation time, outside any test.
+inline const Value* Load(const std::string& file, bool report) {
+ static std::map<std::string, Value> cache;
+ auto it = cache.find(file);
+ if (it != cache.end()) return &it->second;
+ const char* dir = std::getenv("LAYA_GOLDENS");
+ const char* require = std::getenv("LAYA_REQUIRE_GOLDENS");
+ if (dir == nullptr || *dir == 0) {
+ if (report && require != nullptr && std::string(require) == "1") {
+ ADD_FAILURE() << "LAYA_GOLDENS is unset but LAYA_REQUIRE_GOLDENS=1";
+ }
+ return nullptr;
+ }
+ std::ifstream in(std::string(dir) + "/" + file, std::ios::binary);
+ if (!in) {
+ if (report) ADD_FAILURE() << "cannot open golden " << dir << "/" << file;
+ return nullptr;
+ }
+ std::stringstream ss;
+ ss << in.rdbuf();
+ auto parsed = laya::pyvalue::Parse(ss.str());
+ if (!parsed.ok()) {
+ if (report) ADD_FAILURE() << file << ": " << parsed.error();
+ return nullptr;
+ }
+ return &(cache[file] = std::move(parsed).value());
+}
+
+// Walks a path of object keys; nullptr when anything is missing.
+inline const Value* At(const Value* v, std::initializer_list<const char*> path) {
+ for (const char* k : path) {
+ if (v == nullptr) return nullptr;
+ v = v->Find(k);
+ }
+ return v;
+}
+
+// A test body's view of one golden list: fails (or skips locally) when it cannot be had.
+#define LAYA_GOLDEN_LIST(var, file, ...) \
+ const laya::pyvalue::Value* var = laya_test::At(laya_test::Load(file, true), {__VA_ARGS__}); \
+ if (var == nullptr) { \
+ if (!HasFailure()) GTEST_SKIP() << "LAYA_GOLDENS unset (local run)"; \
+ return; \
+ }
+
+constexpr size_t kChunk = 100; // rows per test case in the large families
+inline std::vector<int> Chunks(const std::string& file, std::initializer_list<const char*> path) {
+ const Value* v = At(Load(file, false), path);
+ size_t rows = v == nullptr ? 1 : (v->size() + kChunk - 1) / kChunk;
+ std::vector<int> out;
+ for (size_t i = 0; i < rows; i++) out.push_back(int(i));
+ return out;
+}
+inline std::vector<int> Rows(const std::string& file, std::initializer_list<const char*> path) {
+ const Value* v = At(Load(file, false), path);
+ size_t rows = v == nullptr ? 1 : v->size();
+ std::vector<int> out;
+ for (size_t i = 0; i < rows; i++) out.push_back(int(i));
+ return out;
+}
+inline uint64_t BitsOf(const Value& v) {
+ return v.kind() == Value::Kind::Int ? uint64_t(v.as_int()) : std::strtoull(v.big_int().c_str(), nullptr, 10);
+}
+inline double Num(const Value& v) {
+ return v.kind() == Value::Kind::Int ? double(v.as_int()) : v.as_float();
+}
+
+} // namespace laya_test
diff --git a/laya-go/scripts/gen_tables.py b/laya-go/scripts/gen_tables.py
index 9e814f07..e05a4c59 100644
--- a/laya-go/scripts/gen_tables.py
+++ b/laya-go/scripts/gen_tables.py
@@ -35,6 +35,7 @@ HANGUL = range(0xAC00, 0xD7A4)
OUT_UNICODE = os.path.join(GO, "internal", "unicodedata", "tables_gen.go")
OUT_LANG = os.path.join(GO, "lang", "tables_gen.go")
OUT_TOKENIZER = os.path.join(GO, "tokenizer", "tables_gen.go")
+OUT_CPP_UNICODE = os.path.join(REPO, "laya-cpp", "src", "unicodedata_tables.inc")
def wrap(values, indent="\t", width=100):
@@ -86,7 +87,7 @@ def header(source, extra=""):
# ---------------------------------------------------------------------------------------------
# CPython: the character predicates laya/lang.py leans on.
# ---------------------------------------------------------------------------------------------
-def render_unicode():
+def render_unicode(cpp=False):
word = re.compile(r"[^\W\d_]", re.UNICODE)
digit = re.compile(r"\d", re.UNICODE)
preds = [
@@ -114,8 +115,11 @@ def render_unicode():
"// Version is the Unicode release these tables were recorded from.\n"
"const Version = %r\n\n" % unicodedata.unidata_version]
out[-1] = out[-1].replace("'", '"')
+ pred_tables = []
for name, doc, pred in preds:
- out.append(table(name, ranges(pred), doc))
+ rows = ranges(pred)
+ pred_tables.append((name, rows, pred))
+ out.append(table(name, rows, doc))
out.append(ascii_masks(name, lambda cp, p=pred: p(cp)))
simple, multi = [], []
@@ -173,6 +177,36 @@ def render_unicode():
"// final when they precede it: cased and not case-ignorable."))
out.append(table("caseIgnorable", runs(ignorable),
"Case-ignorable code points, which final-sigma detection skips."))
+ if cpp:
+ return render_unicode_cpp(pred_tables, simple, multi, runs(blocking), runs(ignorable))
+ return "".join(out)
+
+
+def cpp_array(name, values, ctype="uint32_t"):
+ return "inline constexpr %s %s[] = {\n%s\n};\n\n" % (ctype, name, wrap(values, indent=" "))
+
+
+def render_unicode_cpp(pred_tables, simple, multi, cased_rows, ignorable_rows):
+ """The same CPython tables as render_unicode, as a C++ include for laya-cpp/src/unicodedata.cpp."""
+ out = ["// Code generated by laya-go/scripts/gen_tables.py --lang cpp from CPython %d.%d (Unicode %s); "
+ "DO NOT EDIT.\n//\n// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp\n\n"
+ "inline constexpr const char* kUnicodeVersion = \"%s\";\n\n"
+ % (sys.version_info[:2] + (unicodedata.unidata_version, unicodedata.unidata_version))]
+ for name, rows, pred in pred_tables:
+ flat = [v for row in rows for v in row]
+ out.append(cpp_array("k_" + name, flat))
+ low = sum(1 << cp for cp in range(0, 64) if pred(cp))
+ high = sum(1 << (cp - 64) for cp in range(64, 128) if pred(cp))
+ out.append("inline constexpr uint64_t k_%s_ascii[2] = {%#x, %#x};\n\n" % (name, low, high))
+ out.append(cpp_array("k_lower_from", [c for c, _ in simple]))
+ out.append(cpp_array("k_lower_to", [t for _, t in simple]))
+ assert all(len(low) <= 3 for _, low in multi)
+ out.append("struct LowerMulti { uint32_t from; uint32_t count; uint32_t to[3]; };\n"
+ "inline constexpr LowerMulti k_lower_multi[] = {\n%s\n};\n\n"
+ % "\n".join(" {%#x, %d, {%s}}," % (cp, len(low), ", ".join("%#x" % c for c in low + [0] * (3 - len(low))))
+ for cp, low in multi))
+ out.append(cpp_array("k_cased", [v for row in cased_rows for v in row]))
+ out.append(cpp_array("k_case_ignorable", [v for row in ignorable_rows for v in row]))
return "".join(out)
@@ -425,15 +459,19 @@ def render_normalization(normalizers):
TARGETS = ((OUT_UNICODE, render_unicode), (OUT_LANG, render_lang), (OUT_TOKENIZER, render_tokenizer))
+# laya-cpp: the lang tables join in C1 part 2, the tokenizer tables in C2.
+CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)),)
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--check", action="store_true",
help="do not write; exit 1 if a committed copy would change")
+ parser.add_argument("--lang", choices=("go", "cpp"), default="go",
+ help="which port's tables to write or check (default go)")
args = parser.parse_args(argv)
stale = 0
- for path, render in TARGETS:
+ for path, render in (CPP_TARGETS if args.lang == "cpp" else TARGETS):
fresh = render().rstrip("\n") + "\n"
rel = os.path.relpath(path, REPO)
if args.check: