Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(cpp): laya::lang script and language detection
Adeshina Lasisic7e67bee287c
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txtindex 542a7a18..9df26e1e 100644--- a/laya-cpp/CMakeLists.txt+++ b/laya-cpp/CMakeLists.txt@@ -25,7 +25,7 @@ function(laya_target_flags target)target_compile_options(${target} PRIVATE ${LAYA_FP_FLAGS} ${LAYA_WARN_FLAGS})endfunction()-add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp)+add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp src/lang.cpp)target_include_directories(laya PUBLIC include)laya_target_flags(laya)@@ -44,7 +44,7 @@ if(LAYA_BUILD_TESTS)FetchContent_MakeAvailable(googletest)enable_testing()- add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp)+ add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp)target_link_libraries(laya_tests PRIVATE laya GTest::gtest_main)laya_target_flags(laya_tests)diff --git a/laya-cpp/include/laya/lang.hpp b/laya-cpp/include/laya/lang.hppnew file mode 100644index 00000000..7f065c9d--- /dev/null+++ b/laya-cpp/include/laya/lang.hpp@@ -0,0 +1,51 @@+#pragma once+// Script and language detection, a port of laya/lang.py (via laya-go/lang). A state is a+// pyvalue::Value: a string, or lists and objects of them nested up to kMaxDepth levels; numbers,+// booleans and null contribute no text. Objects are read in insertion order, as a Python dict is.+// Every function is safe for concurrent use: the tables are immutable after first use.+#include <string>+#include <string_view>+#include <utility>+#include <vector>++#include "laya/pyvalue.hpp"++namespace laya {+namespace lang {++constexpr int kMaxChars = 4000; // characters of a state that are read+constexpr int kMaxDepth = 6; // container nesting that is read++// Script names are lang.py's strings: "greek", ..., "latin", "other", "unknown".+using Profile = std::vector<std::pair<std::string, double>>; // Latin first, then first appearance++struct LatinProfile {+ std::string language; // "" when undecided+ int english_hits = 0;+ double diacritic_rate = 0;+ bool looks_non_english = false;+};++struct Analysis {+ std::string script;+ Profile script_profile;+ std::string language; // "" when none could be named+ bool english = false;+ double diacritic_rate = 0;+ double non_latin_fraction = 0;+ std::string mixed_segment;+};++// The named non-Latin script of one letter, or "" when none claims it.+std::string ScriptOf(char32_t r);+std::string DetectScript(std::string_view text);+Profile ScriptProfile(std::string_view text);+LatinProfile LatinProfileOf(std::string_view text);+std::string StateText(const pyvalue::Value& state); // leaves joined by a space, at most kMaxChars+Analysis Analyze(const pyvalue::Value& state);+Analysis Analyze(std::string_view text); // a state that is one string+std::string DetectLanguage(const pyvalue::Value& state);+bool IsEnglish(const pyvalue::Value& state);++} // namespace lang+} // namespace layadiff --git a/laya-cpp/src/lang.cpp b/laya-cpp/src/lang.cppnew file mode 100644index 00000000..21ecb79d--- /dev/null+++ b/laya-cpp/src/lang.cpp@@ -0,0 +1,635 @@+#include "laya/lang.hpp"++#include <algorithm>+#include <cmath>+#include <unordered_map>+#include <unordered_set>++#include "laya/pyjson.hpp"+#include "laya/unicodedata.hpp"++namespace laya {+namespace lang {+namespace ud = unicodedata;+namespace {+#include "lang_tables.inc"++using Set = std::unordered_set<std::string>;+constexpr size_t kScripts = sizeof(k_script_names) / sizeof(k_script_names[0]);+constexpr int kShoutedMinStopword = 4, kShoutedMinWord = 5;+constexpr int kLatinBelowCounts = 0x02B0, kLatinBelowScriptOf = 0x0250;++template <size_t N>+Set MakeSet(const char* const (&w)[N]) {+ return Set(w, w + N);+}+struct Tables {+ Set short_swedish = MakeSet(k_short_swedish), shared = MakeSet(k_shared), nordic = MakeSet(k_nordic_overlap),+ en_only = MakeSet(k_en_only), en_collision = MakeSet(k_en_collision), openers{"kan", "jag", "vi"};+ std::vector<std::pair<std::string, Set>> stops;+ Tables() {+ for (const StopSpec& s : k_stop_lists) stops.emplace_back(s.code, Set(s.words, s.words + s.count));+ }+ const Set* Stop(const std::string& code) const {+ for (const auto& s : stops) {+ if (s.first == code) return &s.second;+ }+ return nullptr;+ }+};+const Tables& T() {+ static const Tables t;+ return t;+}++template <class F>+void Each(std::string_view s, F fn) { // fn(byte offset, rune, width)+ for (size_t i = 0; i < s.size();) {+ ud::Decoded d = ud::DecodeRune(s.substr(i));+ fn(i, d.rune, d.width);+ i += d.width;+ }+}+bool Has(const Set& s, const std::string& w) { return s.count(w) != 0; }+bool Intersects(const Set& a, const Set& b) {+ for (const auto& w : a) {+ if (b.count(w)) return true;+ }+ return false;+}+bool IsDiacritic(char32_t r) { return std::binary_search(std::begin(k_non_en_diacritics), std::end(k_non_en_diacritics), uint32_t(r)); }+int CountAlpha(std::string_view t) {+ int n = 0;+ Each(t, [&](size_t, char32_t r, size_t) { n += ud::IsAlpha(r); });+ return n;+}+bool HasDiacritic(const std::string& w) {+ bool h = false;+ Each(w, [&](size_t, char32_t r, size_t) { h = h || IsDiacritic(r); });+ return h;+}+std::vector<std::string> Words(std::string_view text) {+ std::vector<std::string> out;+ long start = -1;+ Each(text, [&](size_t i, char32_t r, size_t) {+ if (ud::IsWord(r)) {+ if (start < 0) start = long(i);+ } else if (start >= 0) {+ out.emplace_back(text.substr(size_t(start), i - size_t(start)));+ start = -1;+ }+ });+ if (start >= 0) out.emplace_back(text.substr(size_t(start)));+ return out;+}+bool IsIdentChar(char32_t r) { return r == '-' || ud::IsPythonWord(r); }+size_t RunEnd(std::string_view t, size_t from) {+ size_t i = from;+ while (i < t.size()) {+ ud::Decoded d = ud::DecodeRune(t.substr(i));+ if (!IsIdentChar(d.rune)) break;+ i += d.width;+ }+ return i;+}+long MatchIdentifier(std::string_view t, size_t from) { // [\w-]*(?:[.@][\w-]+)++ size_t i = RunEnd(t, from);+ int groups = 0;+ while (i < t.size() && (t[i] == '.' || t[i] == '@')) {+ size_t after = RunEnd(t, i + 1);+ if (after == i + 1) break;+ i = after;+ groups++;+ }+ return groups == 0 ? -1 : long(i);+}+std::string SubstituteIdentifiers(std::string_view t) {+ std::string out;+ size_t copied = 0;+ bool replaced = false;+ for (size_t i = 0; i < t.size();) {+ size_t w = ud::DecodeRune(t.substr(i)).width;+ if (i > 0 && IsIdentChar(ud::DecodeLastRune(t.substr(0, i)).rune)) {+ i += w;+ continue;+ }+ long end = MatchIdentifier(t, i);+ if (end < 0) {+ i += w;+ continue;+ }+ replaced = true;+ out.append(t.substr(copied, i - copied));+ out.push_back(' ');+ i = copied = size_t(end);+ }+ if (!replaced) return std::string(t);+ out.append(t.substr(copied));+ return out;+}+bool HasCodeLine(std::string_view t) {+ for (size_t i = 0; i < t.size(); i++) {+ char c = t[i];+ if (c == '=' || c == ';' || c == '{' || c == '}' || c == '[' || c == ']') return true;+ if (c == '(' && i > 0 && ud::IsPythonWord(ud::DecodeLastRune(t.substr(0, i)).rune)) return true;+ }+ return false;+}+bool HasJoined(std::string_view tok) {+ bool found = false;+ Each(tok, [&](size_t i, char32_t r, size_t w) {+ if (found || (r != '.' && r != '_' && r != '/' && r != '\\')) return;+ (void)w;+ if (i == 0 || i + 1 >= tok.size()) return;+ if (ud::IsWordOrDigit(ud::DecodeLastRune(tok.substr(0, i)).rune) && ud::IsWordOrDigit(ud::DecodeRune(tok.substr(i + 1)).rune)) found = true;+ });+ return found;+}+template <class Keep, class Fn>+void EachWordRun(std::string_view text, Keep keep, Fn fn) {+ long start = -1;+ size_t last = 0;+ Each(text, [&](size_t i, char32_t r, size_t) {+ if (ud::IsWord(r)) {+ if (start < 0) {+ if (i > last) keep(text.substr(last, i - last));+ start = long(i);+ }+ } else if (start >= 0) {+ fn(text.substr(size_t(start), i - size_t(start)));+ start = -1;+ last = i;+ }+ });+ if (start >= 0) {+ fn(text.substr(size_t(start)));+ return;+ }+ if (last < text.size()) keep(text.substr(last));+}+std::string BlankUpperRuns(std::string_view text) {+ std::string out;+ EachWordRun(text, [&](std::string_view gap) { out.append(gap); }, [&](std::string_view run) {+ if (ud::RuneCount(run) >= 2 && ud::IsUpperString(run)) out.push_back(' ');+ else out.append(run);+ });+ return out;+}+std::vector<std::string> UpperLetterRuns(std::string_view text) {+ std::vector<std::string> runs;+ EachWordRun(text, [](std::string_view) {}, [&](std::string_view run) {+ if (ud::RuneCount(run) >= 2 && ud::IsUpperString(run)) runs.emplace_back(run);+ });+ return runs;+}+bool EnglishRescued(const std::vector<std::string>& ws, double rate) {+ if (rate >= k_english_rescue_diacritic_rate) return false;+ Set uniq(ws.begin(), ws.end());+ int eo = 0, wd = 0;+ for (const auto& w : uniq) {+ eo += Has(T().en_only, w);+ wd += HasDiacritic(w);+ }+ return eo >= 2 && wd <= 1;+}+bool HasOwnEvidence(const Set& ws, const std::string& language) {+ const Set* stop = T().Stop(language);+ if (!stop) return false;+ for (const auto& w : ws) {+ if (Has(*stop, w) && !Has(T().shared, w)) return true;+ }+ return false;+}+bool Contains(const std::vector<std::string>& ws, const char* t) { return std::find(ws.begin(), ws.end(), t) != ws.end(); }+std::string Head(std::string_view s, long count) {+ if (count <= 0) return "";+ size_t i = 0;+ for (; i < s.size() && count > 0; count--) i += ud::DecodeRune(s.substr(i)).width;+ return std::string(s.substr(0, i));+}+std::vector<std::string> SplitNewline(const std::string& s) {+ std::vector<std::string> out;+ size_t p = 0;+ for (;;) {+ size_t q = s.find('\n', p);+ if (q == std::string::npos) {+ out.push_back(s.substr(p));+ return out;+ }+ out.push_back(s.substr(p, q - p));+ p = q + 1;+ }+}++// ---- scripts -------------------------------------------------------------------------------+bool IsLatinRange(char32_t r, char32_t below) {+ return r < below || (r >= 0x1E00 && r <= 0x1EFF) || (r >= 0xFF21 && r <= 0xFF3A) || (r >= 0xFF41 && r <= 0xFF5A);+}+int LookupScript(char32_t r) {+ if (r > k_script_max_code_point) return -1;+ size_t lo = 0, hi = sizeof(k_script_lookup_index);+ while (lo < hi) {+ size_t mid = lo + (hi - lo) / 2;+ if (r < k_script_lookup[mid * 2]) hi = mid;+ else if (r > k_script_lookup[mid * 2 + 1]) lo = mid + 1;+ else return k_script_lookup_index[mid];+ }+ return -1;+}+int ScriptIndexOf(char32_t r) { return IsLatinRange(r, kLatinBelowScriptOf) ? -1 : LookupScript(r); }++using Counts = std::vector<std::pair<std::string, int>>;+Counts ScriptCounts(std::string_view text) { // Latin comes last, so a named script wins a tie+ int tally[kScripts + 1] = {};+ size_t order[kScripts + 1];+ size_t distinct = 0;+ int latin = 0;+ Each(text, [&](size_t, char32_t r, size_t) {+ if (!ud::IsAlpha(r)) return;+ if (IsLatinRange(r, kLatinBelowCounts)) {+ latin++;+ return;+ }+ int slot = LookupScript(r);+ size_t sl = slot < 0 ? kScripts : size_t(slot);+ if (tally[sl] == 0) order[distinct++] = sl;+ tally[sl]++;+ });+ Counts counts;+ for (size_t k = 0; k < distinct; k++) counts.emplace_back(order[k] == kScripts ? "other" : k_script_names[order[k]], tally[order[k]]);+ counts.emplace_back("latin", latin);+ return counts;+}+std::string ScriptFromCounts(const Counts& c) {+ std::string best;+ int bn = 0;+ for (const auto& x : c) {+ if (x.second > bn) {+ best = x.first;+ bn = x.second;+ }+ }+ return bn == 0 ? "unknown" : best;+}+Profile ProfileFromCounts(const Counts& c) {+ int total = 0;+ for (const auto& x : c) total += x.second;+ Profile p;+ if (total == 0) return p;+ for (const auto& x : c) {+ if (x.first == "latin" && x.second != 0) p.emplace_back(x.first, double(x.second) / double(total));+ }+ for (const auto& x : c) {+ if (x.first != "latin" && x.second != 0) p.emplace_back(x.first, double(x.second) / double(total));+ }+ return p;+}+double Get(const Profile& p, const char* s) {+ for (const auto& x : p) {+ if (x.first == s) return x.second;+ }+ return 0;+}+bool NonLatinCarries(double nl, int letters) {+ return nl >= k_non_latin_fraction || (nl >= k_non_latin_min_fraction && letters >= k_non_latin_min_letters);+}+size_t NonLatinWordCount(std::string_view text) {+ size_t n = 0;+ std::vector<char32_t> cur;+ int script = -1;+ auto emit = [&]() {+ if (cur.size() >= 2 && !ud::IsUpper(cur[0])) n++;+ cur.clear();+ };+ Each(text, [&](size_t, char32_t r, size_t) {+ if (ud::IsCombining(r)) return;+ int here = ScriptIndexOf(r);+ if (here >= 0 && here == script) {+ cur.push_back(r);+ return;+ }+ emit();+ if (here >= 0) cur.push_back(r);+ script = here;+ });+ emit();+ return n;+}++// ---- language ------------------------------------------------------------------------------+bool Shouted(std::string_view text) {+ bool upper = false, lower = false;+ Each(text, [&](size_t, char32_t r, size_t) {+ if (lower) return;+ if (ud::IsLower(r)) lower = true;+ else if (ud::IsUpper(r)) upper = true;+ });+ return !lower && upper;+}+bool ShoutedEvidence(const std::vector<std::string>& tokens, const std::string& language, double rate) {+ bool has_word = false;+ for (const auto& t : tokens) has_word = has_word || ud::RuneCount(t) >= size_t(kShoutedMinWord);+ if (!has_word) return false;+ if (rate >= k_non_en_diacritic_rate) return true;+ const Set* stop = T().Stop(language);+ if (!stop) return false;+ for (const auto& t : tokens) {+ std::string low = ud::Lower(t);+ if (ud::RuneCount(low) >= size_t(kShoutedMinStopword) && Has(*stop, low)) return true;+ }+ return false;+}+std::string NamedProseLanguage(const std::string& segment) {+ if (ud::IsBlank(segment) || HasCodeLine(segment)) return "";+ std::string prose;+ bool first = true;+ for (const auto& tok : ud::Fields(segment)) {+ if (HasJoined(tok)) continue;+ if (!first) prose.push_back(' ');+ prose += tok;+ first = false;+ }+ bool all_caps = Shouted(prose);+ if (!all_caps) prose = BlankUpperRuns(prose);+ auto tokens = Words(prose);+ if (tokens.size() < 4) return "";+ LatinProfile p = LatinProfileOf(prose);+ if (p.language.empty() || p.language == "en") return "";+ if (all_caps && !ShoutedEvidence(tokens, p.language, p.diacritic_rate)) return "";+ const Set* stop = T().Stop(p.language);+ if (!stop) return "";+ Set distinct;+ for (const auto& t : tokens) {+ std::string low = ud::Lower(t);+ if (Has(*stop, low)) distinct.insert(low);+ }+ return distinct.size() < 2 ? "" : p.language;+}++std::string DominantNonLatin(const Profile& p) {+ std::string best;+ double share = 0;+ for (const auto& s : p) {+ if (s.first == "latin") continue;+ if (best.empty() || s.second > share) {+ best = s.first;+ share = s.second;+ }+ }+ return best;+}++Analysis AnalyzeText(const std::string& text) {+ Counts counts = ScriptCounts(text);+ Profile profile = ProfileFromCounts(counts);+ std::string script = ScriptFromCounts(counts);+ double nl = 0;+ if (!profile.empty()) nl = pyjson::Round4(1.0 - Get(profile, "latin"));+ int nl_letters = int(std::nearbyint(nl * double(CountAlpha(text))));+ if (script == "latin" && NonLatinWordCount(text) > 0 && NonLatinCarries(nl, nl_letters)) script = DominantNonLatin(profile);+ Analysis a;+ a.script_profile = profile;+ if (script == "unknown") {+ a.script = "unknown";+ a.english = true;+ return a;+ }+ a.script = script;+ if (script != "latin") {+ a.non_latin_fraction = nl;+ return a;+ }+ LatinProfile latin = LatinProfileOf(text);+ std::string language = latin.language;+ if (!language.empty() && language != "en" && !NonLatinCarries(nl, nl_letters)) {+ if (Shouted(text)) {+ if (!ShoutedEvidence(Words(text), language, latin.diacritic_rate)) language = "";+ } else {+ auto caps = UpperLetterRuns(text);+ bool all_acronyms = !caps.empty();+ for (const auto& run : caps) {+ if (ud::RuneCount(run) >= size_t(kShoutedMinWord)) {+ all_acronyms = false;+ break;+ }+ }+ if (all_acronyms) {+ std::string blanked = BlankUpperRuns(text);+ if (blanked != text) language = LatinProfileOf(blanked).language;+ }+ }+ }+ a.language = language;+ a.english = language == "en" || (language.empty() && !latin.looks_non_english);+ a.diacritic_rate = pyjson::Round4(latin.diacritic_rate);+ a.non_latin_fraction = nl;+ return a;+}++bool LeafNonEnglish(const std::string& leaf, Analysis& best_out) {+ int best_letters = -1;+ Analysis best;+ for (const auto& line : SplitNewline(leaf)) {+ if (ud::RuneCount(line) < 7) continue;+ std::string sample = Head(line, kMaxChars);+ if (ud::IsBlank(sample) || HasCodeLine(sample)) continue;+ Analysis d = AnalyzeText(sample);+ if (d.english) continue;+ bool named = !d.language.empty() && d.language != "en" && !NamedProseLanguage(sample).empty();+ if (!named) {+ if (d.script != "latin" && d.script != "unknown") {+ if (NonLatinWordCount(sample) == 0 || CountAlpha(sample) < k_non_latin_min_letters) continue;+ } else if (!(d.diacritic_rate >= k_non_en_diacritic_rate && Words(sample).size() >= 4)) {+ continue;+ }+ }+ int letters = CountAlpha(sample);+ if (letters > best_letters) {+ best_letters = letters;+ best = d;+ }+ }+ best_out = best;+ return best_letters >= 0;+}++// ---- state ---------------------------------------------------------------------------------+struct Collector {+ std::vector<std::string> out;+ bool limited, done;+ long budget;+ Collector(long limit, bool lim) : limited(lim), done(lim && limit <= 0), budget(limit) {}+ void Leaf(const std::string& s) {+ if (done) return;+ if (!limited) {+ out.push_back(s);+ return;+ }+ long n = long(ud::RuneCount(s));+ if (n > budget) {+ out.push_back(Head(s, budget));+ done = true;+ return;+ }+ out.push_back(s);+ budget -= n + 1;+ if (budget <= 0) done = true;+ }+};+void Collect(const pyvalue::Value& v, int depth, Collector& c) {+ if (depth > kMaxDepth || c.done) return;+ switch (v.kind()) {+ case pyvalue::Value::Kind::String: c.Leaf(ud::ToValid(v.as_string())); break;+ case pyvalue::Value::Kind::Array:+ case pyvalue::Value::Kind::Object:+ for (size_t i = 0; i < v.size() && !c.done; i++) Collect(v.value(i), depth + 1, c);+ break;+ default: break;+ }+}+std::string JoinLeaves(const std::vector<std::string>& leaves, long max_chars) {+ Collector c(max_chars, true);+ for (const auto& l : leaves) c.Leaf(l);+ std::string joined;+ for (size_t i = 0; i < c.out.size(); i++) {+ if (i) joined.push_back(' ');+ joined += c.out[i];+ }+ return Head(joined, max_chars);+}+} // namespace++std::string ScriptOf(char32_t r) {+ int i = ScriptIndexOf(r);+ return i < 0 ? "" : k_script_names[i];+}++std::string DetectScript(std::string_view text) { return ScriptFromCounts(ScriptCounts(ud::ToValid(text))); }+Profile ScriptProfile(std::string_view text) { return ProfileFromCounts(ScriptCounts(ud::ToValid(text))); }++LatinProfile LatinProfileOf(std::string_view raw) {+ const std::string text = ud::ToValid(raw);+ std::string sub = SubstituteIdentifiers(text);+ for (size_t p = sub.find("\xC4\xB0"); p != std::string::npos; p = sub.find("\xC4\xB0", p + 1)) sub.replace(p, 2, "i");+ std::vector<std::string> ws = Words(ud::Lower(sub));+ std::string lowered = ud::Lower(text);+ int diacritics = 0;+ Each(lowered, [&](size_t, char32_t r, size_t) { diacritics += IsDiacritic(r); });+ size_t n = std::max<size_t>(ud::RuneCount(lowered), 1);+ double rate = double(diacritics) / double(n);+ bool non_english = rate >= k_non_en_diacritic_rate;+ Set uniq(ws.begin(), ws.end());+ bool nordic = Intersects(uniq, T().nordic) && !Intersects(uniq, T().en_only);+ LatinProfile out;+ out.diacritic_rate = rate;+ if (ws.size() > 1 && ws.size() < 4 && Intersects(uniq, T().short_swedish)) {+ out.language = "sv";+ out.looks_non_english = non_english;+ return out;+ }+ if (ws.size() < 4) {+ out.looks_non_english = non_english || nordic;+ return out;+ }+ std::unordered_map<std::string, int> counts;+ for (const auto& w : ws) counts[w]++;+ const auto& stops = T().stops;+ std::vector<int> scores(stops.size());+ int english = 0;+ for (size_t i = 0; i < stops.size(); i++) {+ int score = 0;+ for (const auto& kv : counts) {+ if (Has(stops[i].second, kv.first)) score += Has(T().en_collision, kv.first) ? 1 : kv.second;+ }+ scores[i] = score;+ if (stops[i].first == "en") english = score;+ }+ std::string best_language;+ int best = 0;+ bool seen = false;+ for (size_t i = 0; i < stops.size(); i++) {+ const std::string& code = stops[i].first;+ if (code == "en" || !HasOwnEvidence(uniq, code)) continue;+ if (!seen || scores[i] > best) { // strictly greater: the first listed language wins a tie+ best_language = code;+ best = scores[i];+ seen = true;+ }+ }+ std::string language;+ if (seen && best >= std::max(2, english + 2)) {+ language = best_language;+ } else if (best_language == "sv" && Contains(ws, "inte") && Contains(ws, "kan") && Has(T().openers, ws[0]) && english <= 1) {+ language = best_language;+ } else if (seen && non_english && best >= std::max(2, english)) {+ language = best_language;+ } else if (english != 0 && (!non_english || EnglishRescued(ws, rate))) {+ language = "en";+ }+ out.language = language;+ out.english_hits = english;+ out.looks_non_english = non_english || (language.empty() && nordic);+ return out;+}++std::string StateText(const pyvalue::Value& state) {+ Collector c(kMaxChars, true);+ Collect(state, 0, c);+ return JoinLeaves(c.out, kMaxChars);+}++Analysis Analyze(const pyvalue::Value& state) {+ Collector all(0, false);+ Collect(state, 0, all);+ const std::vector<std::string>& leaves = all.out;+ Analysis result = AnalyzeText(JoinLeaves(leaves, kMaxChars));+ if (result.script == "latin" && result.english) {+ bool multiline = false;+ for (const auto& l : leaves) multiline = multiline || l.find('\n') != std::string::npos;+ if (leaves.size() > 1 || multiline) {+ long seen = 0;+ bool found = false;+ for (const auto& leaf : leaves) {+ for (const auto& line : SplitNewline(leaf)) {+ if (seen >= kMaxChars) goto done_segment;+ std::string seg = Head(line, kMaxChars - seen);+ seen += long(ud::RuneCount(seg));+ std::string lg = NamedProseLanguage(seg);+ if (!lg.empty()) {+ result.language = lg;+ result.english = false;+ result.mixed_segment = ud::Strip(seg);+ found = true;+ goto done_segment;+ }+ }+ }+ done_segment:+ (void)found;+ }+ }+ const bool scalar = state.kind() == pyvalue::Value::Kind::String || state.is_null();+ if (scalar || !result.english) return result;+ long best_letters = -1;+ Analysis best;+ for (const auto& leaf : leaves) {+ Analysis d;+ if (!LeafNonEnglish(leaf, d)) continue;+ long letters = CountAlpha(Head(leaf, kMaxChars));+ if (letters > best_letters) {+ best_letters = letters;+ best = d;+ }+ }+ if (best_letters < 0) return result;+ result.language = best.language;+ result.english = false;+ return result;+}++Analysis Analyze(std::string_view text) { return Analyze(pyvalue::Value::String(std::string(text))); }+std::string DetectLanguage(const pyvalue::Value& s) { return Analyze(s).language; }+bool IsEnglish(const pyvalue::Value& s) { return Analyze(s).english; }++} // namespace lang+} // namespace layadiff --git a/laya-cpp/src/lang_tables.inc b/laya-cpp/src/lang_tables.incnew file mode 100644index 00000000..fae10959--- /dev/null+++ b/laya-cpp/src/lang_tables.inc@@ -0,0 +1,191 @@+// Code generated by laya-go/scripts/gen_tables.py --lang cpp from laya/lang.py; DO NOT EDIT.+//+// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp++inline constexpr const char* k_script_names[] = {"greek", "cyrillic", "armenian", "hebrew", "arabic", "devanagari", "bengali", "gurmukhi", "gujarati", "oriya", "tamil", "telugu", "kannada", "malayalam", "sinhala", "thai", "lao", "tibetan", "myanmar", "georgian", "ethiopic", "khmer", "hangul", "kana", "han"};++inline constexpr uint32_t k_script_lookup[] = {+ 880, 1023, 1024, 1327, 1328, 1423, 1424, 1535, 1536, 1791, 1872, 1919, 2208, 2303, 2304, 2431,+ 2432, 2559, 2560, 2687, 2688, 2815, 2816, 2943, 2944, 3071, 3072, 3199, 3200, 3327, 3328, 3455,+ 3456, 3583, 3584, 3711, 3712, 3839, 3840, 4095, 4096, 4255, 4256, 4351, 4352, 4607, 4608, 4991,+ 6016, 6143, 7936, 8191, 11744, 11775, 12352, 12447, 12448, 12543, 12592, 12687, 12784, 12799,+ 13312, 19903, 19968, 40959, 42560, 42655, 43232, 43263, 44032, 55215, 63744, 64255, 64336,+ 65023, 65136, 65279,+};++inline constexpr uint8_t k_script_lookup_index[] = {+ 0, 1, 2, 3, 4, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 22, 20, 21, 0, 1,+ 23, 23, 22, 23, 24, 24, 1, 5, 22, 24, 4, 4,+};++inline constexpr uint32_t k_script_max_code_point = 0xfeff;++inline constexpr const char* k_stop_0[] = {+ "after", "already", "always", "and", "any", "are", "aren", "as", "at", "be", "because", "been",+ "before", "being", "between", "both", "but", "can", "could", "couldn", "didn", "didnt", "does",+ "doesn", "doesnt", "doing", "done", "dont", "down", "during", "each", "ever", "every", "few",+ "for", "from", "has", "hasn", "have", "haven", "havent", "here", "him", "his", "how", "i",+ "if", "in", "into", "is", "isn", "isnt", "it", "ive", "many", "might", "much", "must", "never",+ "nor", "not", "of", "on", "only", "onto", "our", "please", "shall", "she", "should", "some",+ "than", "that", "thats", "the", "their", "them", "then", "there", "these", "they", "this",+ "those", "through", "to", "too", "until", "very", "was", "wasn", "wasnt", "we", "were",+ "weren", "what", "whats", "when", "where", "which", "while", "who", "whom", "why", "will",+ "with", "within", "without", "won", "wont", "would", "wouldn", "yet", "you", "your", "youre",+ "yours",+};++inline constexpr const char* k_stop_1[] = {+ "alors", "au", "aux", "avec", "bien", "bonjour", "ce", "ces", "cette", "comment", "dans",+ "des", "deux", "dois", "doit", "donc", "du", "elle", "elles", "est", "et", "fait", "fois",+ "il", "ils", "je", "jour", "jours", "la", "le", "les", "ma", "mais", "merci", "mes", "mois",+ "mon", "nous", "ont", "ou", "pas", "peut", "peux", "plus", "pour", "pourquoi", "quand", "que",+ "qui", "sa", "ses", "sont", "sur", "ta", "tes", "ton", "tous", "tout", "toute", "trois",+ "très", "tu", "une", "veut", "veux", "vous", "être",+};++inline constexpr const char* k_stop_2[] = {+ "aber", "auch", "auf", "aus", "bei", "bitte", "das", "dem", "den", "der", "dich", "die",+ "diese", "diesen", "dieser", "dieses", "dir", "ein", "eine", "einem", "einen", "einer", "für",+ "gibt", "habe", "haben", "heute", "ich", "im", "in", "ist", "jetzt", "kann", "kannst", "mein",+ "meine", "meinem", "meinen", "meiner", "mich", "mir", "mit", "nach", "nicht", "noch", "oder",+ "sich", "sind", "und", "uns", "von", "wann", "was", "welche", "werden", "wie", "wir", "wird",+ "wo", "wurde", "zu", "zum", "zur",+};++inline constexpr const char* k_stop_3[] = {+ "al", "algo", "aquí", "aunque", "como", "con", "cuando", "del", "donde", "dos", "el", "entre",+ "es", "esa", "ese", "eso", "esta", "este", "esto", "está", "fue", "fueron", "gracias", "han",+ "hay", "hemos", "hoy", "la", "las", "le", "les", "lo", "los", "mi", "muy", "más", "nada",+ "necesito", "ni", "nos", "para", "pero", "por", "porque", "puede", "pueden", "que", "quiero",+ "se", "ser", "sobre", "son", "su", "sus", "también", "tengo", "tiene", "tienen", "todo",+ "tres", "tu", "un", "una", "y", "ya",+};++inline constexpr const char* k_stop_4[] = {+ "agora", "ainda", "alguem", "alguém", "ali", "antes", "ao", "aos", "aqui", "as", "até", "boa",+ "cadê", "com", "como", "consigo", "da", "das", "depois", "deu", "do", "dois", "dos", "e", "em",+ "entao", "então", "era", "esta", "estamos", "estava", "este", "estou", "está", "eu", "ficou",+ "fiz", "foi", "gostaria", "hoje", "isso", "isto", "ja", "já", "mais", "mas", "meu", "meus",+ "minha", "minhas", "muito", "na", "nada", "nao", "nas", "nenhum", "nenhuma", "ninguem",+ "ninguém", "noite", "nos", "nossa", "nosso", "não", "o", "obrigada", "obrigado", "olá", "onde",+ "ontem", "os", "para", "pela", "pelo", "pode", "podem", "por", "porque", "pra", "preciso",+ "quando", "que", "quero", "sao", "se", "ser", "seu", "sou", "sua", "são", "tambem", "também",+ "tarde", "tem", "tenho", "três", "tudo", "tá", "um", "uma", "vc", "vcs", "voce", "voces",+ "você", "vocês", "é",+};++inline constexpr const char* k_stop_5[] = {+ "abbiamo", "adesso", "agli", "alla", "alle", "anche", "ancora", "avete", "che", "ci", "ciao",+ "col", "come", "con", "da", "dagli", "dal", "dalla", "dallo", "degli", "dei", "del", "della",+ "delle", "dello", "deve", "devo", "devono", "di", "dove", "e", "ed", "era", "fra", "già",+ "gli", "grazie", "ha", "hai", "hanno", "ho", "ieri", "il", "la", "le", "lo", "mai", "mi",+ "mia", "mio", "molto", "ne", "negli", "nel", "nell", "nella", "non", "o", "oggi", "per",+ "perche", "più", "poco", "quando", "questa", "questo", "scusa", "sempre", "si", "sono",+ "stata", "stato", "su", "sua", "sul", "sulla", "sulle", "tra", "tuo", "un", "una", "uno",+ "voglio", "vorrei", "è",+};++inline constexpr const char* k_stop_6[] = {+ "aan", "dat", "deze", "door", "een", "het", "is", "maar", "met", "naar", "niet", "ook", "op",+ "te", "van", "voor", "worden", "wordt", "zijn",+};++inline constexpr const char* k_stop_7[] = {+ "appen", "aterbetala", "aterbetalning", "att", "behover", "behöver", "blivit", "debiterade",+ "dessa", "detta", "faktura", "fel", "från", "får", "ganger", "gång", "gånger", "gör", "göra",+ "hittar", "hjalp", "hjalpa", "hjälp", "hjälpa", "här", "installningarna", "installningen",+ "inställningarna", "inställningen", "inte", "jag", "kommer", "kraschar", "kvittot", "losenord",+ "lösenord", "mig", "min", "mitt", "måste", "nar", "när", "nästa", "och", "också", "om",+ "oppnar", "paket", "ska", "skicka", "skulle", "sparningen", "spårningen", "säger", "till",+ "tva", "två", "upp", "uppdaterats", "utan", "vi", "vill", "är", "återbetala", "återbetalning",+ "öppnar",+};++inline constexpr const char* k_stop_8[] = {+ "aceasta", "această", "acest", "acesta", "acum", "ale", "care", "dar", "din", "după", "este",+ "foarte", "fost", "fără", "lui", "mi", "nu", "pentru", "până", "sunt", "să", "trebuie",+ "vreau", "vă", "în", "și", "ți",+};++inline constexpr const char* k_stop_9[] = {+ "abar", "ajke", "akhon", "amader", "amake", "amar", "ami", "amra", "apnake", "apnar", "apnara",+ "apni", "ar", "asbe", "bhai", "bhalo", "bolte", "bolun", "chai", "chaina", "dhonnobad",+ "dilam", "dite", "diye", "diyechi", "dorkar", "duibar", "ei", "eita", "ekbar", "ekhon",+ "ekhono", "ekta", "ferot", "geche", "gese", "hobe", "hocche", "hoise", "hoye", "hoyeche",+ "hoyni", "jabe", "jodi", "jonno", "kalke", "keno", "keu", "kharap", "khub", "ki", "kibhabe",+ "kichu", "kintu", "kivabe", "kobe", "kokhon", "korbo", "korchi", "kore", "koreche", "korechi",+ "koren", "korlam", "korsi", "korte", "korun", "kothay", "koto", "lagbe", "moddhe", "nai",+ "niye", "oi", "oita", "onek", "ota", "pabo", "paini", "parben", "parbo", "parchi", "parchina",+ "peyechi", "sathe", "shathe", "shob", "shomossa", "somossa", "tader", "tahole", "taka",+ "theke", "tomake", "tomar", "tomra", "tumi", "valo",+};++inline constexpr const char* k_stop_10[] = {+ "amma", "ancaq", "artiq", "artıq", "bir", "biz", "bu", "cox", "daha", "deyil", "də", "eger",+ "görə", "həm", "hər", "ile", "ilə", "isə", "kimi", "lakin", "mən", "nə", "olan", "olmasa",+ "olub", "onlar", "siz", "sonra", "sən", "ucun", "var", "ve", "və", "yalniz", "yalnız", "yox",+ "yoxdur", "çox", "üçün", "əgər",+};++struct StopSpec { const char* code; const char* const* words; size_t count; };+// In lang.py declaration order: the language that wins a tied score is the first listed.+inline constexpr StopSpec k_stop_lists[] = {+ {"en", k_stop_0, 116},+ {"fr", k_stop_1, 67},+ {"de", k_stop_2, 63},+ {"es", k_stop_3, 65},+ {"pt", k_stop_4, 107},+ {"it", k_stop_5, 85},+ {"nl", k_stop_6, 19},+ {"sv", k_stop_7, 66},+ {"ro", k_stop_8, 27},+ {"bn", k_stop_9, 98},+ {"az", k_stop_10, 40},+};++inline constexpr const char* k_short_swedish[] = {+ "abonnemanget", "aterbetalning", "atkomst", "behover", "behöver", "betalningen", "fakturan",+ "felmeddelande", "glomt", "glömt", "inloggningen", "installningen", "inställningen", "kvitto",+ "kvittot", "losenord", "lösenord", "sparningen", "spårningen", "återbetalning", "åtkomst",+};++inline constexpr const char* k_shared[] = {+ "as", "como", "con", "da", "das", "del", "dois", "dos", "e", "era", "esta", "este", "está",+ "får", "hej", "il", "in", "is", "ja", "jo", "kommer", "la", "le", "les", "lo", "mais", "mi",+ "mig", "min", "nada", "nej", "nos", "o", "om", "para", "por", "porque", "quando", "que", "se",+ "ser", "skulle", "su", "sua", "tack", "tu", "un", "una", "vi", "was",+};++inline constexpr const char* k_nordic_overlap[] = {+ "får", "hej", "ja", "jo", "kommer", "mig", "min", "nej", "om", "skulle", "tack", "vi",+};++inline constexpr const char* k_en_only[] = {+ "after", "already", "always", "and", "any", "are", "aren", "at", "be", "because", "been",+ "before", "being", "between", "both", "but", "can", "could", "couldn", "didn", "didnt", "does",+ "doesn", "doesnt", "doing", "done", "dont", "down", "during", "each", "ever", "every", "few",+ "for", "from", "has", "hasn", "have", "haven", "havent", "here", "him", "his", "how", "i",+ "if", "into", "isn", "isnt", "it", "ive", "many", "might", "much", "must", "never", "nor",+ "not", "of", "on", "only", "onto", "our", "please", "shall", "she", "should", "some", "than",+ "that", "thats", "the", "their", "them", "then", "there", "these", "they", "this", "those",+ "through", "to", "too", "until", "very", "wasn", "wasnt", "we", "were", "weren", "what",+ "whats", "when", "where", "which", "while", "who", "whom", "why", "will", "with", "within",+ "without", "won", "wont", "would", "wouldn", "yet", "you", "your", "youre", "yours",+};++inline constexpr const char* k_en_collision[] = {+ "care", "come", "do", "im", "per", "plus", "son", "todo",+};++inline constexpr uint32_t k_non_en_diacritics[] = {+ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 241, 242,+ 243, 244, 245, 246, 248, 249, 250, 251, 252, 253, 255, 257, 259, 261, 263, 269, 271, 273, 275,+ 281, 283, 287, 291, 299, 305, 311, 316, 322, 324, 326, 328, 337, 339, 345, 347, 351, 353, 355,+ 357, 363, 367, 369, 378, 380, 382, 537, 539, 601,+};++inline constexpr double k_non_en_diacritic_rate = 0.02;+inline constexpr double k_english_rescue_diacritic_rate = 0.06;+inline constexpr double k_non_latin_fraction = 0.2;+inline constexpr double k_non_latin_min_fraction = 0.1;+inline constexpr int k_non_latin_min_letters = 10;diff --git a/laya-cpp/tests/lang_test.cpp b/laya-cpp/tests/lang_test.cppnew file mode 100644index 00000000..9842dcd6--- /dev/null+++ b/laya-cpp/tests/lang_test.cpp@@ -0,0 +1,125 @@+#include <cmath>++#include "golden.hpp"+#include "laya/lang.hpp"+#include "laya/sha256.hpp"+#include "laya/unicodedata.hpp"++using laya_test::At;+using laya_test::Value;+namespace lg = laya::lang;+namespace pv = laya::pyvalue;++namespace {+// The state a golden row describes.+pv::Value StateOf(const Value& enc) {+ const std::string& kind = enc.Find("kind")->as_string();+ const Value& v = *enc.Find("value");+ if (kind == "bytes") {+ std::string b;+ for (size_t i = 0; i < v.size(); i++) b.push_back(char(v.value(i).as_int()));+ return laya::unicodedata::ValidUtf8(b) ? pv::Value::String(b) : pv::Value::Null(); // undecodable bytes read as nothing+ }+ return v;+}+void ExpectProfile(const Value& want, const lg::Profile& got, const std::string& what) {+ ASSERT_EQ(want.size(), got.size()) << what;+ for (size_t i = 0; i < got.size(); i++) {+ EXPECT_EQ(want.key(i), got[i].first) << what;+ EXPECT_EQ(laya_test::Num(want.value(i)), got[i].second) << what << " " << got[i].first;+ }+}+} // namespace++class DetectRow : public ::testing::TestWithParam<int> {};+TEST_P(DetectRow, MatchesPython) {+ LAYA_GOLDEN_LIST(rows, "lang_detect.json", "cases");+ const Value& row = rows->value(size_t(GetParam()));+ const std::string name = row.Find("name")->as_string();+ pv::Value state = StateOf(*row.Find("state"));+ const std::string text = row.Find("state_text")->as_string();+ EXPECT_EQ(lg::StateText(state), text) << name;+ EXPECT_EQ(lg::DetectScript(text), row.Find("detect_script")->as_string()) << name;+ ExpectProfile(*row.Find("script_profile"), lg::ScriptProfile(text), name);+ const Value& lp = *row.Find("latin_profile");+ lg::LatinProfile got = lg::LatinProfileOf(text);+ auto wl = lp.Find("language");+ EXPECT_EQ(got.language, wl->is_null() ? "" : wl->as_string()) << name;+ EXPECT_EQ(got.english_hits, lp.Find("english_hits")->as_int()) << name;+ EXPECT_EQ(got.diacritic_rate, laya_test::Num(*lp.Find("diacritic_rate"))) << name;+ EXPECT_EQ(got.looks_non_english, lp.Find("looks_non_english")->as_bool()) << name;+ const Value& an = *row.Find("analyse");+ lg::Analysis a = lg::Analyze(state);+ EXPECT_EQ(a.script, an.Find("script")->as_string()) << name;+ ExpectProfile(*an.Find("script_profile"), a.script_profile, name);+ auto al = an.Find("language");+ EXPECT_EQ(a.language, al->is_null() ? "" : al->as_string()) << name;+ EXPECT_EQ(a.english, an.Find("is_english")->as_bool()) << name;+ EXPECT_EQ(a.diacritic_rate, laya_test::Num(*an.Find("diacritic_rate"))) << name;+ EXPECT_EQ(a.non_latin_fraction, laya_test::Num(*an.Find("non_latin_fraction"))) << name;+ auto ms = an.Find("mixed_segment");+ EXPECT_EQ(a.mixed_segment, ms->is_null() ? "" : ms->as_string()) << name;+ EXPECT_EQ(lg::IsEnglish(state), row.Find("is_english")->as_bool()) << name;+ EXPECT_EQ(lg::DetectLanguage(state), a.language) << name;+}+INSTANTIATE_TEST_SUITE_P(Golden, DetectRow, ::testing::ValuesIn(laya_test::Rows("lang_detect.json", {"cases"})));++TEST(Lang, ScriptOfMatchesPythonEverywhere) {+ LAYA_GOLDEN_LIST(want, "lang_detect.json", "unicode_digests", "script_of");+ laya::Sha256 h;+ for (char32_t r = 0; r < 0x110000; r++) h.Update(lg::ScriptOf(r) + ";");+ EXPECT_EQ(laya::ToHex(h.Finish()), want->as_string());+}++TEST(Lang, StateShapesMatchPython) {+ LAYA_GOLDEN_LIST(shapes, "go_extra.json", "state_shapes");+ const std::string german = shapes->Find("string")->Find("state_text")->as_string();+ auto S = [](const std::string& s) { return pv::Value::String(s); };+ auto L = [](std::vector<pv::Value> v) { return pv::Value::Array(std::move(v)); };+ auto O = [](const char* k, pv::Value v) {+ pv::Value o = pv::Value::Object();+ o.Set(k, std::move(v));+ return o;+ };+ pv::Value seven = S(german), six = S(german);+ for (int i = 0; i < 7; i++) seven = L({seven});+ for (int i = 0; i < 6; i++) six = L({six});+ std::vector<std::pair<std::string, pv::Value>> built = {+ {"string", S(german)},+ {"map_of_slices", O("a", L({S(german)}))},+ {"slice_of_maps", L({O("a", S(german))})},+ {"map_of_objects", O("a", O("b", S(german)))},+ {"slice_of_slices", L({L({S(german)})})},+ {"tuple", L({S(german)})},+ {"seven_deep", seven},+ {"six_deep", six},+ {"numbers_and_none", L({pv::Value::Int(1), pv::Value::Float(2.5), pv::Value::Null(), pv::Value::Bool(true)})},+ {"mixed", O("k", L({S(german), pv::Value::Int(3), O("x", pv::Value::Null())}))},+ {"bytes", S(german)},+ {"invalid_bytes", pv::Value::Null()}};+ for (const auto& b : built) {+ const Value* want = shapes->Find(b.first);+ ASSERT_NE(want, nullptr) << b.first;+ lg::Analysis a = lg::Analyze(b.second);+ EXPECT_EQ(lg::StateText(b.second), want->Find("state_text")->as_string()) << b.first;+ EXPECT_EQ(a.english, want->Find("is_english")->as_bool()) << b.first;+ auto wl = want->Find("language");+ EXPECT_EQ(a.language, wl->is_null() ? "" : wl->as_string()) << b.first;+ EXPECT_EQ(a.script, want->Find("script")->as_string()) << b.first;+ }+}++TEST(Lang, InvalidUtf8MatchesPython) {+ LAYA_GOLDEN_LIST(fam, "go_extra.json", "invalid_utf8");+ const Value& hex = *fam->Find("hex");+ const Value& script = *fam->Find("script");+ const Value& text = *fam->Find("state_text");+ ASSERT_EQ(hex.size(), 3000u);+ for (size_t i = 0; i < hex.size(); i++) {+ const std::string& h = hex.value(i).as_string();+ std::string bytes;+ for (size_t k = 0; k + 1 < h.size(); k += 2) bytes.push_back(char(std::stoi(h.substr(k, 2), nullptr, 16)));+ EXPECT_EQ(lg::DetectScript(bytes), script.value(i).as_string()) << "row " << i;+ EXPECT_EQ(lg::StateText(pv::Value::String(bytes)), text.value(i).as_string()) << "row " << i;+ }+}diff --git a/laya-go/scripts/gen_tables.py b/laya-go/scripts/gen_tables.pyindex e05a4c59..9aa3a986 100644--- a/laya-go/scripts/gen_tables.py+++ b/laya-go/scripts/gen_tables.py@@ -36,6 +36,7 @@ OUT_UNICODE = os.path.join(GO, "internal", "unicodedata", "tables_gen.go")OUT_LANG = os.path.join(GO, "lang", "tables_gen.go")OUT_TOKENIZER = os.path.join(GO, "tokenizer", "tables_gen.go")OUT_CPP_UNICODE = os.path.join(REPO, "laya-cpp", "src", "unicodedata_tables.inc")+OUT_CPP_LANG = os.path.join(REPO, "laya-cpp", "src", "lang_tables.inc")def wrap(values, indent="\t", width=100):@@ -280,6 +281,49 @@ def render_lang():return "".join(out)+def render_lang_cpp():+ """laya.lang's script ranges, function words and thresholds, as a C++ include for src/lang.cpp."""+ sys.path.insert(0, REPO)+ from laya import lang++ flat = sorted((lo, hi, name) for name, rs in lang._SCRIPT_RANGES for lo, hi in rs)+ for (a_lo, a_hi, a_name), (b_lo, b_hi, b_name) in zip(flat, flat[1:]):+ if a_hi >= b_lo:+ raise SystemExit("gen_tables: script ranges %s and %s overlap" % (a_name, b_name))+ names = [name for name, _ in lang._SCRIPT_RANGES]+ index_of = {name: n for n, name in enumerate(names)}++ def words(name, ws):+ return "inline constexpr const char* %s[] = {\n%s\n};\n\n" % (+ name, wrap([go_str(w) for w in sorted(ws)], indent=" "))++ out = ["// Code generated by laya-go/scripts/gen_tables.py --lang cpp from laya/lang.py; DO NOT EDIT.\n"+ "//\n// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp\n\n"]+ out.append("inline constexpr const char* k_script_names[] = {%s};\n\n" % ", ".join(go_str(n) for n in names))+ out.append(cpp_array("k_script_lookup", [v for lo, hi, _ in flat for v in (lo, hi)]))+ out.append(cpp_array("k_script_lookup_index", [index_of[n] for _, _, n in flat], "uint8_t"))+ out.append("inline constexpr uint32_t k_script_max_code_point = %#x;\n\n" % max(hi for _, hi, _ in flat))+ stops = list(lang._STOP.items())+ for n, (code, ws) in enumerate(stops):+ out.append(words("k_stop_%d" % n, ws))+ out.append("struct StopSpec { const char* code; const char* const* words; size_t count; };\n"+ "// In lang.py declaration order: the language that wins a tied score is the first listed.\n"+ "inline constexpr StopSpec k_stop_lists[] = {\n%s\n};\n\n"+ % "\n".join(" {%s, k_stop_%d, %d}," % (go_str(c), n, len(ws)) for n, (c, ws) in enumerate(stops)))+ for name, ws in (("k_short_swedish", lang._SHORT_SWEDISH_WORDS), ("k_shared", lang._SHARED_WORDS),+ ("k_nordic_overlap", lang._NORDIC_OVERLAP_WORDS), ("k_en_only", lang._EN_ONLY_WORDS),+ ("k_en_collision", lang._EN_COLLISION_WORDS)):+ out.append(words(name, ws))+ out.append(cpp_array("k_non_en_diacritics", sorted(ord(c) for c in lang._NON_EN_DIACRITICS)))+ for name, value in (("k_non_en_diacritic_rate", lang.NON_EN_DIACRITIC_RATE),+ ("k_english_rescue_diacritic_rate", lang.ENGLISH_RESCUE_DIACRITIC_RATE),+ ("k_non_latin_fraction", lang.NON_LATIN_FRACTION),+ ("k_non_latin_min_fraction", lang.NON_LATIN_MIN_FRACTION)):+ out.append("inline constexpr double %s = %r;\n" % (name, value))+ out.append("inline constexpr int k_non_latin_min_letters = %d;\n" % lang.NON_LATIN_MIN_LETTERS)+ return "".join(out)++# ---------------------------------------------------------------------------------------------# The tokenizers crate: pre-tokenizer classes, single_word flank, lowercase, normalization.# ---------------------------------------------------------------------------------------------@@ -459,8 +503,8 @@ def render_normalization(normalizers):TARGETS = ((OUT_UNICODE, render_unicode), (OUT_LANG, render_lang), (OUT_TOKENIZER, render_tokenizer))-# laya-cpp: the lang tables join in C1 part 2, the tokenizer tables in C2.-CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)),)+# laya-cpp: the tokenizer tables join in C2.+CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)), (OUT_CPP_LANG, render_lang_cpp))def main(argv=None):