Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(cpp): laya::lang script and language detection

Adeshina Lasisic7e67bee287c
diff --git a/laya-cpp/CMakeLists.txt b/laya-cpp/CMakeLists.txt
index 542a7a18..9df26e1e 100644
--- a/laya-cpp/CMakeLists.txt
+++ b/laya-cpp/CMakeLists.txt
@@ -25,7 +25,7 @@ function(laya_target_flags target)
target_compile_options(${target} PRIVATE ${LAYA_FP_FLAGS} ${LAYA_WARN_FLAGS})
endfunction()
-add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp)
+add_library(laya src/version.cpp src/sha256.cpp src/unicodedata.cpp src/pyjson.cpp src/pyvalue.cpp src/lang.cpp)
target_include_directories(laya PUBLIC include)
laya_target_flags(laya)
@@ -44,7 +44,7 @@ if(LAYA_BUILD_TESTS)
FetchContent_MakeAvailable(googletest)
enable_testing()
- add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp)
+ add_executable(laya_tests tests/version_test.cpp tests/foundation_test.cpp tests/lang_test.cpp)
target_link_libraries(laya_tests PRIVATE laya GTest::gtest_main)
laya_target_flags(laya_tests)
diff --git a/laya-cpp/include/laya/lang.hpp b/laya-cpp/include/laya/lang.hpp
new file mode 100644
index 00000000..7f065c9d
--- /dev/null
+++ b/laya-cpp/include/laya/lang.hpp
@@ -0,0 +1,51 @@
+#pragma once
+// Script and language detection, a port of laya/lang.py (via laya-go/lang). A state is a
+// pyvalue::Value: a string, or lists and objects of them nested up to kMaxDepth levels; numbers,
+// booleans and null contribute no text. Objects are read in insertion order, as a Python dict is.
+// Every function is safe for concurrent use: the tables are immutable after first use.
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include "laya/pyvalue.hpp"
+
+namespace laya {
+namespace lang {
+
+constexpr int kMaxChars = 4000; // characters of a state that are read
+constexpr int kMaxDepth = 6; // container nesting that is read
+
+// Script names are lang.py's strings: "greek", ..., "latin", "other", "unknown".
+using Profile = std::vector<std::pair<std::string, double>>; // Latin first, then first appearance
+
+struct LatinProfile {
+ std::string language; // "" when undecided
+ int english_hits = 0;
+ double diacritic_rate = 0;
+ bool looks_non_english = false;
+};
+
+struct Analysis {
+ std::string script;
+ Profile script_profile;
+ std::string language; // "" when none could be named
+ bool english = false;
+ double diacritic_rate = 0;
+ double non_latin_fraction = 0;
+ std::string mixed_segment;
+};
+
+// The named non-Latin script of one letter, or "" when none claims it.
+std::string ScriptOf(char32_t r);
+std::string DetectScript(std::string_view text);
+Profile ScriptProfile(std::string_view text);
+LatinProfile LatinProfileOf(std::string_view text);
+std::string StateText(const pyvalue::Value& state); // leaves joined by a space, at most kMaxChars
+Analysis Analyze(const pyvalue::Value& state);
+Analysis Analyze(std::string_view text); // a state that is one string
+std::string DetectLanguage(const pyvalue::Value& state);
+bool IsEnglish(const pyvalue::Value& state);
+
+} // namespace lang
+} // namespace laya
diff --git a/laya-cpp/src/lang.cpp b/laya-cpp/src/lang.cpp
new file mode 100644
index 00000000..21ecb79d
--- /dev/null
+++ b/laya-cpp/src/lang.cpp
@@ -0,0 +1,635 @@
+#include "laya/lang.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "laya/pyjson.hpp"
+#include "laya/unicodedata.hpp"
+
+namespace laya {
+namespace lang {
+namespace ud = unicodedata;
+namespace {
+#include "lang_tables.inc"
+
+using Set = std::unordered_set<std::string>;
+constexpr size_t kScripts = sizeof(k_script_names) / sizeof(k_script_names[0]);
+constexpr int kShoutedMinStopword = 4, kShoutedMinWord = 5;
+constexpr int kLatinBelowCounts = 0x02B0, kLatinBelowScriptOf = 0x0250;
+
+template <size_t N>
+Set MakeSet(const char* const (&w)[N]) {
+ return Set(w, w + N);
+}
+struct Tables {
+ Set short_swedish = MakeSet(k_short_swedish), shared = MakeSet(k_shared), nordic = MakeSet(k_nordic_overlap),
+ en_only = MakeSet(k_en_only), en_collision = MakeSet(k_en_collision), openers{"kan", "jag", "vi"};
+ std::vector<std::pair<std::string, Set>> stops;
+ Tables() {
+ for (const StopSpec& s : k_stop_lists) stops.emplace_back(s.code, Set(s.words, s.words + s.count));
+ }
+ const Set* Stop(const std::string& code) const {
+ for (const auto& s : stops) {
+ if (s.first == code) return &s.second;
+ }
+ return nullptr;
+ }
+};
+const Tables& T() {
+ static const Tables t;
+ return t;
+}
+
+template <class F>
+void Each(std::string_view s, F fn) { // fn(byte offset, rune, width)
+ for (size_t i = 0; i < s.size();) {
+ ud::Decoded d = ud::DecodeRune(s.substr(i));
+ fn(i, d.rune, d.width);
+ i += d.width;
+ }
+}
+bool Has(const Set& s, const std::string& w) { return s.count(w) != 0; }
+bool Intersects(const Set& a, const Set& b) {
+ for (const auto& w : a) {
+ if (b.count(w)) return true;
+ }
+ return false;
+}
+bool IsDiacritic(char32_t r) { return std::binary_search(std::begin(k_non_en_diacritics), std::end(k_non_en_diacritics), uint32_t(r)); }
+int CountAlpha(std::string_view t) {
+ int n = 0;
+ Each(t, [&](size_t, char32_t r, size_t) { n += ud::IsAlpha(r); });
+ return n;
+}
+bool HasDiacritic(const std::string& w) {
+ bool h = false;
+ Each(w, [&](size_t, char32_t r, size_t) { h = h || IsDiacritic(r); });
+ return h;
+}
+std::vector<std::string> Words(std::string_view text) {
+ std::vector<std::string> out;
+ long start = -1;
+ Each(text, [&](size_t i, char32_t r, size_t) {
+ if (ud::IsWord(r)) {
+ if (start < 0) start = long(i);
+ } else if (start >= 0) {
+ out.emplace_back(text.substr(size_t(start), i - size_t(start)));
+ start = -1;
+ }
+ });
+ if (start >= 0) out.emplace_back(text.substr(size_t(start)));
+ return out;
+}
+bool IsIdentChar(char32_t r) { return r == '-' || ud::IsPythonWord(r); }
+size_t RunEnd(std::string_view t, size_t from) {
+ size_t i = from;
+ while (i < t.size()) {
+ ud::Decoded d = ud::DecodeRune(t.substr(i));
+ if (!IsIdentChar(d.rune)) break;
+ i += d.width;
+ }
+ return i;
+}
+long MatchIdentifier(std::string_view t, size_t from) { // [\w-]*(?:[.@][\w-]+)+
+ size_t i = RunEnd(t, from);
+ int groups = 0;
+ while (i < t.size() && (t[i] == '.' || t[i] == '@')) {
+ size_t after = RunEnd(t, i + 1);
+ if (after == i + 1) break;
+ i = after;
+ groups++;
+ }
+ return groups == 0 ? -1 : long(i);
+}
+std::string SubstituteIdentifiers(std::string_view t) {
+ std::string out;
+ size_t copied = 0;
+ bool replaced = false;
+ for (size_t i = 0; i < t.size();) {
+ size_t w = ud::DecodeRune(t.substr(i)).width;
+ if (i > 0 && IsIdentChar(ud::DecodeLastRune(t.substr(0, i)).rune)) {
+ i += w;
+ continue;
+ }
+ long end = MatchIdentifier(t, i);
+ if (end < 0) {
+ i += w;
+ continue;
+ }
+ replaced = true;
+ out.append(t.substr(copied, i - copied));
+ out.push_back(' ');
+ i = copied = size_t(end);
+ }
+ if (!replaced) return std::string(t);
+ out.append(t.substr(copied));
+ return out;
+}
+bool HasCodeLine(std::string_view t) {
+ for (size_t i = 0; i < t.size(); i++) {
+ char c = t[i];
+ if (c == '=' || c == ';' || c == '{' || c == '}' || c == '[' || c == ']') return true;
+ if (c == '(' && i > 0 && ud::IsPythonWord(ud::DecodeLastRune(t.substr(0, i)).rune)) return true;
+ }
+ return false;
+}
+bool HasJoined(std::string_view tok) {
+ bool found = false;
+ Each(tok, [&](size_t i, char32_t r, size_t w) {
+ if (found || (r != '.' && r != '_' && r != '/' && r != '\\')) return;
+ (void)w;
+ if (i == 0 || i + 1 >= tok.size()) return;
+ if (ud::IsWordOrDigit(ud::DecodeLastRune(tok.substr(0, i)).rune) && ud::IsWordOrDigit(ud::DecodeRune(tok.substr(i + 1)).rune)) found = true;
+ });
+ return found;
+}
+template <class Keep, class Fn>
+void EachWordRun(std::string_view text, Keep keep, Fn fn) {
+ long start = -1;
+ size_t last = 0;
+ Each(text, [&](size_t i, char32_t r, size_t) {
+ if (ud::IsWord(r)) {
+ if (start < 0) {
+ if (i > last) keep(text.substr(last, i - last));
+ start = long(i);
+ }
+ } else if (start >= 0) {
+ fn(text.substr(size_t(start), i - size_t(start)));
+ start = -1;
+ last = i;
+ }
+ });
+ if (start >= 0) {
+ fn(text.substr(size_t(start)));
+ return;
+ }
+ if (last < text.size()) keep(text.substr(last));
+}
+std::string BlankUpperRuns(std::string_view text) {
+ std::string out;
+ EachWordRun(text, [&](std::string_view gap) { out.append(gap); }, [&](std::string_view run) {
+ if (ud::RuneCount(run) >= 2 && ud::IsUpperString(run)) out.push_back(' ');
+ else out.append(run);
+ });
+ return out;
+}
+std::vector<std::string> UpperLetterRuns(std::string_view text) {
+ std::vector<std::string> runs;
+ EachWordRun(text, [](std::string_view) {}, [&](std::string_view run) {
+ if (ud::RuneCount(run) >= 2 && ud::IsUpperString(run)) runs.emplace_back(run);
+ });
+ return runs;
+}
+bool EnglishRescued(const std::vector<std::string>& ws, double rate) {
+ if (rate >= k_english_rescue_diacritic_rate) return false;
+ Set uniq(ws.begin(), ws.end());
+ int eo = 0, wd = 0;
+ for (const auto& w : uniq) {
+ eo += Has(T().en_only, w);
+ wd += HasDiacritic(w);
+ }
+ return eo >= 2 && wd <= 1;
+}
+bool HasOwnEvidence(const Set& ws, const std::string& language) {
+ const Set* stop = T().Stop(language);
+ if (!stop) return false;
+ for (const auto& w : ws) {
+ if (Has(*stop, w) && !Has(T().shared, w)) return true;
+ }
+ return false;
+}
+bool Contains(const std::vector<std::string>& ws, const char* t) { return std::find(ws.begin(), ws.end(), t) != ws.end(); }
+std::string Head(std::string_view s, long count) {
+ if (count <= 0) return "";
+ size_t i = 0;
+ for (; i < s.size() && count > 0; count--) i += ud::DecodeRune(s.substr(i)).width;
+ return std::string(s.substr(0, i));
+}
+std::vector<std::string> SplitNewline(const std::string& s) {
+ std::vector<std::string> out;
+ size_t p = 0;
+ for (;;) {
+ size_t q = s.find('\n', p);
+ if (q == std::string::npos) {
+ out.push_back(s.substr(p));
+ return out;
+ }
+ out.push_back(s.substr(p, q - p));
+ p = q + 1;
+ }
+}
+
+// ---- scripts -------------------------------------------------------------------------------
+bool IsLatinRange(char32_t r, char32_t below) {
+ return r < below || (r >= 0x1E00 && r <= 0x1EFF) || (r >= 0xFF21 && r <= 0xFF3A) || (r >= 0xFF41 && r <= 0xFF5A);
+}
+int LookupScript(char32_t r) {
+ if (r > k_script_max_code_point) return -1;
+ size_t lo = 0, hi = sizeof(k_script_lookup_index);
+ while (lo < hi) {
+ size_t mid = lo + (hi - lo) / 2;
+ if (r < k_script_lookup[mid * 2]) hi = mid;
+ else if (r > k_script_lookup[mid * 2 + 1]) lo = mid + 1;
+ else return k_script_lookup_index[mid];
+ }
+ return -1;
+}
+int ScriptIndexOf(char32_t r) { return IsLatinRange(r, kLatinBelowScriptOf) ? -1 : LookupScript(r); }
+
+using Counts = std::vector<std::pair<std::string, int>>;
+Counts ScriptCounts(std::string_view text) { // Latin comes last, so a named script wins a tie
+ int tally[kScripts + 1] = {};
+ size_t order[kScripts + 1];
+ size_t distinct = 0;
+ int latin = 0;
+ Each(text, [&](size_t, char32_t r, size_t) {
+ if (!ud::IsAlpha(r)) return;
+ if (IsLatinRange(r, kLatinBelowCounts)) {
+ latin++;
+ return;
+ }
+ int slot = LookupScript(r);
+ size_t sl = slot < 0 ? kScripts : size_t(slot);
+ if (tally[sl] == 0) order[distinct++] = sl;
+ tally[sl]++;
+ });
+ Counts counts;
+ for (size_t k = 0; k < distinct; k++) counts.emplace_back(order[k] == kScripts ? "other" : k_script_names[order[k]], tally[order[k]]);
+ counts.emplace_back("latin", latin);
+ return counts;
+}
+std::string ScriptFromCounts(const Counts& c) {
+ std::string best;
+ int bn = 0;
+ for (const auto& x : c) {
+ if (x.second > bn) {
+ best = x.first;
+ bn = x.second;
+ }
+ }
+ return bn == 0 ? "unknown" : best;
+}
+Profile ProfileFromCounts(const Counts& c) {
+ int total = 0;
+ for (const auto& x : c) total += x.second;
+ Profile p;
+ if (total == 0) return p;
+ for (const auto& x : c) {
+ if (x.first == "latin" && x.second != 0) p.emplace_back(x.first, double(x.second) / double(total));
+ }
+ for (const auto& x : c) {
+ if (x.first != "latin" && x.second != 0) p.emplace_back(x.first, double(x.second) / double(total));
+ }
+ return p;
+}
+double Get(const Profile& p, const char* s) {
+ for (const auto& x : p) {
+ if (x.first == s) return x.second;
+ }
+ return 0;
+}
+bool NonLatinCarries(double nl, int letters) {
+ return nl >= k_non_latin_fraction || (nl >= k_non_latin_min_fraction && letters >= k_non_latin_min_letters);
+}
+size_t NonLatinWordCount(std::string_view text) {
+ size_t n = 0;
+ std::vector<char32_t> cur;
+ int script = -1;
+ auto emit = [&]() {
+ if (cur.size() >= 2 && !ud::IsUpper(cur[0])) n++;
+ cur.clear();
+ };
+ Each(text, [&](size_t, char32_t r, size_t) {
+ if (ud::IsCombining(r)) return;
+ int here = ScriptIndexOf(r);
+ if (here >= 0 && here == script) {
+ cur.push_back(r);
+ return;
+ }
+ emit();
+ if (here >= 0) cur.push_back(r);
+ script = here;
+ });
+ emit();
+ return n;
+}
+
+// ---- language ------------------------------------------------------------------------------
+bool Shouted(std::string_view text) {
+ bool upper = false, lower = false;
+ Each(text, [&](size_t, char32_t r, size_t) {
+ if (lower) return;
+ if (ud::IsLower(r)) lower = true;
+ else if (ud::IsUpper(r)) upper = true;
+ });
+ return !lower && upper;
+}
+bool ShoutedEvidence(const std::vector<std::string>& tokens, const std::string& language, double rate) {
+ bool has_word = false;
+ for (const auto& t : tokens) has_word = has_word || ud::RuneCount(t) >= size_t(kShoutedMinWord);
+ if (!has_word) return false;
+ if (rate >= k_non_en_diacritic_rate) return true;
+ const Set* stop = T().Stop(language);
+ if (!stop) return false;
+ for (const auto& t : tokens) {
+ std::string low = ud::Lower(t);
+ if (ud::RuneCount(low) >= size_t(kShoutedMinStopword) && Has(*stop, low)) return true;
+ }
+ return false;
+}
+std::string NamedProseLanguage(const std::string& segment) {
+ if (ud::IsBlank(segment) || HasCodeLine(segment)) return "";
+ std::string prose;
+ bool first = true;
+ for (const auto& tok : ud::Fields(segment)) {
+ if (HasJoined(tok)) continue;
+ if (!first) prose.push_back(' ');
+ prose += tok;
+ first = false;
+ }
+ bool all_caps = Shouted(prose);
+ if (!all_caps) prose = BlankUpperRuns(prose);
+ auto tokens = Words(prose);
+ if (tokens.size() < 4) return "";
+ LatinProfile p = LatinProfileOf(prose);
+ if (p.language.empty() || p.language == "en") return "";
+ if (all_caps && !ShoutedEvidence(tokens, p.language, p.diacritic_rate)) return "";
+ const Set* stop = T().Stop(p.language);
+ if (!stop) return "";
+ Set distinct;
+ for (const auto& t : tokens) {
+ std::string low = ud::Lower(t);
+ if (Has(*stop, low)) distinct.insert(low);
+ }
+ return distinct.size() < 2 ? "" : p.language;
+}
+
+std::string DominantNonLatin(const Profile& p) {
+ std::string best;
+ double share = 0;
+ for (const auto& s : p) {
+ if (s.first == "latin") continue;
+ if (best.empty() || s.second > share) {
+ best = s.first;
+ share = s.second;
+ }
+ }
+ return best;
+}
+
+Analysis AnalyzeText(const std::string& text) {
+ Counts counts = ScriptCounts(text);
+ Profile profile = ProfileFromCounts(counts);
+ std::string script = ScriptFromCounts(counts);
+ double nl = 0;
+ if (!profile.empty()) nl = pyjson::Round4(1.0 - Get(profile, "latin"));
+ int nl_letters = int(std::nearbyint(nl * double(CountAlpha(text))));
+ if (script == "latin" && NonLatinWordCount(text) > 0 && NonLatinCarries(nl, nl_letters)) script = DominantNonLatin(profile);
+ Analysis a;
+ a.script_profile = profile;
+ if (script == "unknown") {
+ a.script = "unknown";
+ a.english = true;
+ return a;
+ }
+ a.script = script;
+ if (script != "latin") {
+ a.non_latin_fraction = nl;
+ return a;
+ }
+ LatinProfile latin = LatinProfileOf(text);
+ std::string language = latin.language;
+ if (!language.empty() && language != "en" && !NonLatinCarries(nl, nl_letters)) {
+ if (Shouted(text)) {
+ if (!ShoutedEvidence(Words(text), language, latin.diacritic_rate)) language = "";
+ } else {
+ auto caps = UpperLetterRuns(text);
+ bool all_acronyms = !caps.empty();
+ for (const auto& run : caps) {
+ if (ud::RuneCount(run) >= size_t(kShoutedMinWord)) {
+ all_acronyms = false;
+ break;
+ }
+ }
+ if (all_acronyms) {
+ std::string blanked = BlankUpperRuns(text);
+ if (blanked != text) language = LatinProfileOf(blanked).language;
+ }
+ }
+ }
+ a.language = language;
+ a.english = language == "en" || (language.empty() && !latin.looks_non_english);
+ a.diacritic_rate = pyjson::Round4(latin.diacritic_rate);
+ a.non_latin_fraction = nl;
+ return a;
+}
+
+bool LeafNonEnglish(const std::string& leaf, Analysis& best_out) {
+ int best_letters = -1;
+ Analysis best;
+ for (const auto& line : SplitNewline(leaf)) {
+ if (ud::RuneCount(line) < 7) continue;
+ std::string sample = Head(line, kMaxChars);
+ if (ud::IsBlank(sample) || HasCodeLine(sample)) continue;
+ Analysis d = AnalyzeText(sample);
+ if (d.english) continue;
+ bool named = !d.language.empty() && d.language != "en" && !NamedProseLanguage(sample).empty();
+ if (!named) {
+ if (d.script != "latin" && d.script != "unknown") {
+ if (NonLatinWordCount(sample) == 0 || CountAlpha(sample) < k_non_latin_min_letters) continue;
+ } else if (!(d.diacritic_rate >= k_non_en_diacritic_rate && Words(sample).size() >= 4)) {
+ continue;
+ }
+ }
+ int letters = CountAlpha(sample);
+ if (letters > best_letters) {
+ best_letters = letters;
+ best = d;
+ }
+ }
+ best_out = best;
+ return best_letters >= 0;
+}
+
+// ---- state ---------------------------------------------------------------------------------
+struct Collector {
+ std::vector<std::string> out;
+ bool limited, done;
+ long budget;
+ Collector(long limit, bool lim) : limited(lim), done(lim && limit <= 0), budget(limit) {}
+ void Leaf(const std::string& s) {
+ if (done) return;
+ if (!limited) {
+ out.push_back(s);
+ return;
+ }
+ long n = long(ud::RuneCount(s));
+ if (n > budget) {
+ out.push_back(Head(s, budget));
+ done = true;
+ return;
+ }
+ out.push_back(s);
+ budget -= n + 1;
+ if (budget <= 0) done = true;
+ }
+};
+void Collect(const pyvalue::Value& v, int depth, Collector& c) {
+ if (depth > kMaxDepth || c.done) return;
+ switch (v.kind()) {
+ case pyvalue::Value::Kind::String: c.Leaf(ud::ToValid(v.as_string())); break;
+ case pyvalue::Value::Kind::Array:
+ case pyvalue::Value::Kind::Object:
+ for (size_t i = 0; i < v.size() && !c.done; i++) Collect(v.value(i), depth + 1, c);
+ break;
+ default: break;
+ }
+}
+std::string JoinLeaves(const std::vector<std::string>& leaves, long max_chars) {
+ Collector c(max_chars, true);
+ for (const auto& l : leaves) c.Leaf(l);
+ std::string joined;
+ for (size_t i = 0; i < c.out.size(); i++) {
+ if (i) joined.push_back(' ');
+ joined += c.out[i];
+ }
+ return Head(joined, max_chars);
+}
+} // namespace
+
+std::string ScriptOf(char32_t r) {
+ int i = ScriptIndexOf(r);
+ return i < 0 ? "" : k_script_names[i];
+}
+
+std::string DetectScript(std::string_view text) { return ScriptFromCounts(ScriptCounts(ud::ToValid(text))); }
+Profile ScriptProfile(std::string_view text) { return ProfileFromCounts(ScriptCounts(ud::ToValid(text))); }
+
+LatinProfile LatinProfileOf(std::string_view raw) {
+ const std::string text = ud::ToValid(raw);
+ std::string sub = SubstituteIdentifiers(text);
+ for (size_t p = sub.find("\xC4\xB0"); p != std::string::npos; p = sub.find("\xC4\xB0", p + 1)) sub.replace(p, 2, "i");
+ std::vector<std::string> ws = Words(ud::Lower(sub));
+ std::string lowered = ud::Lower(text);
+ int diacritics = 0;
+ Each(lowered, [&](size_t, char32_t r, size_t) { diacritics += IsDiacritic(r); });
+ size_t n = std::max<size_t>(ud::RuneCount(lowered), 1);
+ double rate = double(diacritics) / double(n);
+ bool non_english = rate >= k_non_en_diacritic_rate;
+ Set uniq(ws.begin(), ws.end());
+ bool nordic = Intersects(uniq, T().nordic) && !Intersects(uniq, T().en_only);
+ LatinProfile out;
+ out.diacritic_rate = rate;
+ if (ws.size() > 1 && ws.size() < 4 && Intersects(uniq, T().short_swedish)) {
+ out.language = "sv";
+ out.looks_non_english = non_english;
+ return out;
+ }
+ if (ws.size() < 4) {
+ out.looks_non_english = non_english || nordic;
+ return out;
+ }
+ std::unordered_map<std::string, int> counts;
+ for (const auto& w : ws) counts[w]++;
+ const auto& stops = T().stops;
+ std::vector<int> scores(stops.size());
+ int english = 0;
+ for (size_t i = 0; i < stops.size(); i++) {
+ int score = 0;
+ for (const auto& kv : counts) {
+ if (Has(stops[i].second, kv.first)) score += Has(T().en_collision, kv.first) ? 1 : kv.second;
+ }
+ scores[i] = score;
+ if (stops[i].first == "en") english = score;
+ }
+ std::string best_language;
+ int best = 0;
+ bool seen = false;
+ for (size_t i = 0; i < stops.size(); i++) {
+ const std::string& code = stops[i].first;
+ if (code == "en" || !HasOwnEvidence(uniq, code)) continue;
+ if (!seen || scores[i] > best) { // strictly greater: the first listed language wins a tie
+ best_language = code;
+ best = scores[i];
+ seen = true;
+ }
+ }
+ std::string language;
+ if (seen && best >= std::max(2, english + 2)) {
+ language = best_language;
+ } else if (best_language == "sv" && Contains(ws, "inte") && Contains(ws, "kan") && Has(T().openers, ws[0]) && english <= 1) {
+ language = best_language;
+ } else if (seen && non_english && best >= std::max(2, english)) {
+ language = best_language;
+ } else if (english != 0 && (!non_english || EnglishRescued(ws, rate))) {
+ language = "en";
+ }
+ out.language = language;
+ out.english_hits = english;
+ out.looks_non_english = non_english || (language.empty() && nordic);
+ return out;
+}
+
+std::string StateText(const pyvalue::Value& state) {
+ Collector c(kMaxChars, true);
+ Collect(state, 0, c);
+ return JoinLeaves(c.out, kMaxChars);
+}
+
+Analysis Analyze(const pyvalue::Value& state) {
+ Collector all(0, false);
+ Collect(state, 0, all);
+ const std::vector<std::string>& leaves = all.out;
+ Analysis result = AnalyzeText(JoinLeaves(leaves, kMaxChars));
+ if (result.script == "latin" && result.english) {
+ bool multiline = false;
+ for (const auto& l : leaves) multiline = multiline || l.find('\n') != std::string::npos;
+ if (leaves.size() > 1 || multiline) {
+ long seen = 0;
+ bool found = false;
+ for (const auto& leaf : leaves) {
+ for (const auto& line : SplitNewline(leaf)) {
+ if (seen >= kMaxChars) goto done_segment;
+ std::string seg = Head(line, kMaxChars - seen);
+ seen += long(ud::RuneCount(seg));
+ std::string lg = NamedProseLanguage(seg);
+ if (!lg.empty()) {
+ result.language = lg;
+ result.english = false;
+ result.mixed_segment = ud::Strip(seg);
+ found = true;
+ goto done_segment;
+ }
+ }
+ }
+ done_segment:
+ (void)found;
+ }
+ }
+ const bool scalar = state.kind() == pyvalue::Value::Kind::String || state.is_null();
+ if (scalar || !result.english) return result;
+ long best_letters = -1;
+ Analysis best;
+ for (const auto& leaf : leaves) {
+ Analysis d;
+ if (!LeafNonEnglish(leaf, d)) continue;
+ long letters = CountAlpha(Head(leaf, kMaxChars));
+ if (letters > best_letters) {
+ best_letters = letters;
+ best = d;
+ }
+ }
+ if (best_letters < 0) return result;
+ result.language = best.language;
+ result.english = false;
+ return result;
+}
+
+Analysis Analyze(std::string_view text) { return Analyze(pyvalue::Value::String(std::string(text))); }
+std::string DetectLanguage(const pyvalue::Value& s) { return Analyze(s).language; }
+bool IsEnglish(const pyvalue::Value& s) { return Analyze(s).english; }
+
+} // namespace lang
+} // namespace laya
diff --git a/laya-cpp/src/lang_tables.inc b/laya-cpp/src/lang_tables.inc
new file mode 100644
index 00000000..fae10959
--- /dev/null
+++ b/laya-cpp/src/lang_tables.inc
@@ -0,0 +1,191 @@
+// Code generated by laya-go/scripts/gen_tables.py --lang cpp from laya/lang.py; DO NOT EDIT.
+//
+// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp
+
+inline constexpr const char* k_script_names[] = {"greek", "cyrillic", "armenian", "hebrew", "arabic", "devanagari", "bengali", "gurmukhi", "gujarati", "oriya", "tamil", "telugu", "kannada", "malayalam", "sinhala", "thai", "lao", "tibetan", "myanmar", "georgian", "ethiopic", "khmer", "hangul", "kana", "han"};
+
+inline constexpr uint32_t k_script_lookup[] = {
+ 880, 1023, 1024, 1327, 1328, 1423, 1424, 1535, 1536, 1791, 1872, 1919, 2208, 2303, 2304, 2431,
+ 2432, 2559, 2560, 2687, 2688, 2815, 2816, 2943, 2944, 3071, 3072, 3199, 3200, 3327, 3328, 3455,
+ 3456, 3583, 3584, 3711, 3712, 3839, 3840, 4095, 4096, 4255, 4256, 4351, 4352, 4607, 4608, 4991,
+ 6016, 6143, 7936, 8191, 11744, 11775, 12352, 12447, 12448, 12543, 12592, 12687, 12784, 12799,
+ 13312, 19903, 19968, 40959, 42560, 42655, 43232, 43263, 44032, 55215, 63744, 64255, 64336,
+ 65023, 65136, 65279,
+};
+
+inline constexpr uint8_t k_script_lookup_index[] = {
+ 0, 1, 2, 3, 4, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 22, 20, 21, 0, 1,
+ 23, 23, 22, 23, 24, 24, 1, 5, 22, 24, 4, 4,
+};
+
+inline constexpr uint32_t k_script_max_code_point = 0xfeff;
+
+inline constexpr const char* k_stop_0[] = {
+ "after", "already", "always", "and", "any", "are", "aren", "as", "at", "be", "because", "been",
+ "before", "being", "between", "both", "but", "can", "could", "couldn", "didn", "didnt", "does",
+ "doesn", "doesnt", "doing", "done", "dont", "down", "during", "each", "ever", "every", "few",
+ "for", "from", "has", "hasn", "have", "haven", "havent", "here", "him", "his", "how", "i",
+ "if", "in", "into", "is", "isn", "isnt", "it", "ive", "many", "might", "much", "must", "never",
+ "nor", "not", "of", "on", "only", "onto", "our", "please", "shall", "she", "should", "some",
+ "than", "that", "thats", "the", "their", "them", "then", "there", "these", "they", "this",
+ "those", "through", "to", "too", "until", "very", "was", "wasn", "wasnt", "we", "were",
+ "weren", "what", "whats", "when", "where", "which", "while", "who", "whom", "why", "will",
+ "with", "within", "without", "won", "wont", "would", "wouldn", "yet", "you", "your", "youre",
+ "yours",
+};
+
+inline constexpr const char* k_stop_1[] = {
+ "alors", "au", "aux", "avec", "bien", "bonjour", "ce", "ces", "cette", "comment", "dans",
+ "des", "deux", "dois", "doit", "donc", "du", "elle", "elles", "est", "et", "fait", "fois",
+ "il", "ils", "je", "jour", "jours", "la", "le", "les", "ma", "mais", "merci", "mes", "mois",
+ "mon", "nous", "ont", "ou", "pas", "peut", "peux", "plus", "pour", "pourquoi", "quand", "que",
+ "qui", "sa", "ses", "sont", "sur", "ta", "tes", "ton", "tous", "tout", "toute", "trois",
+ "très", "tu", "une", "veut", "veux", "vous", "être",
+};
+
+inline constexpr const char* k_stop_2[] = {
+ "aber", "auch", "auf", "aus", "bei", "bitte", "das", "dem", "den", "der", "dich", "die",
+ "diese", "diesen", "dieser", "dieses", "dir", "ein", "eine", "einem", "einen", "einer", "für",
+ "gibt", "habe", "haben", "heute", "ich", "im", "in", "ist", "jetzt", "kann", "kannst", "mein",
+ "meine", "meinem", "meinen", "meiner", "mich", "mir", "mit", "nach", "nicht", "noch", "oder",
+ "sich", "sind", "und", "uns", "von", "wann", "was", "welche", "werden", "wie", "wir", "wird",
+ "wo", "wurde", "zu", "zum", "zur",
+};
+
+inline constexpr const char* k_stop_3[] = {
+ "al", "algo", "aquí", "aunque", "como", "con", "cuando", "del", "donde", "dos", "el", "entre",
+ "es", "esa", "ese", "eso", "esta", "este", "esto", "está", "fue", "fueron", "gracias", "han",
+ "hay", "hemos", "hoy", "la", "las", "le", "les", "lo", "los", "mi", "muy", "más", "nada",
+ "necesito", "ni", "nos", "para", "pero", "por", "porque", "puede", "pueden", "que", "quiero",
+ "se", "ser", "sobre", "son", "su", "sus", "también", "tengo", "tiene", "tienen", "todo",
+ "tres", "tu", "un", "una", "y", "ya",
+};
+
+inline constexpr const char* k_stop_4[] = {
+ "agora", "ainda", "alguem", "alguém", "ali", "antes", "ao", "aos", "aqui", "as", "até", "boa",
+ "cadê", "com", "como", "consigo", "da", "das", "depois", "deu", "do", "dois", "dos", "e", "em",
+ "entao", "então", "era", "esta", "estamos", "estava", "este", "estou", "está", "eu", "ficou",
+ "fiz", "foi", "gostaria", "hoje", "isso", "isto", "ja", "já", "mais", "mas", "meu", "meus",
+ "minha", "minhas", "muito", "na", "nada", "nao", "nas", "nenhum", "nenhuma", "ninguem",
+ "ninguém", "noite", "nos", "nossa", "nosso", "não", "o", "obrigada", "obrigado", "olá", "onde",
+ "ontem", "os", "para", "pela", "pelo", "pode", "podem", "por", "porque", "pra", "preciso",
+ "quando", "que", "quero", "sao", "se", "ser", "seu", "sou", "sua", "são", "tambem", "também",
+ "tarde", "tem", "tenho", "três", "tudo", "tá", "um", "uma", "vc", "vcs", "voce", "voces",
+ "você", "vocês", "é",
+};
+
+inline constexpr const char* k_stop_5[] = {
+ "abbiamo", "adesso", "agli", "alla", "alle", "anche", "ancora", "avete", "che", "ci", "ciao",
+ "col", "come", "con", "da", "dagli", "dal", "dalla", "dallo", "degli", "dei", "del", "della",
+ "delle", "dello", "deve", "devo", "devono", "di", "dove", "e", "ed", "era", "fra", "già",
+ "gli", "grazie", "ha", "hai", "hanno", "ho", "ieri", "il", "la", "le", "lo", "mai", "mi",
+ "mia", "mio", "molto", "ne", "negli", "nel", "nell", "nella", "non", "o", "oggi", "per",
+ "perche", "più", "poco", "quando", "questa", "questo", "scusa", "sempre", "si", "sono",
+ "stata", "stato", "su", "sua", "sul", "sulla", "sulle", "tra", "tuo", "un", "una", "uno",
+ "voglio", "vorrei", "è",
+};
+
+inline constexpr const char* k_stop_6[] = {
+ "aan", "dat", "deze", "door", "een", "het", "is", "maar", "met", "naar", "niet", "ook", "op",
+ "te", "van", "voor", "worden", "wordt", "zijn",
+};
+
+inline constexpr const char* k_stop_7[] = {
+ "appen", "aterbetala", "aterbetalning", "att", "behover", "behöver", "blivit", "debiterade",
+ "dessa", "detta", "faktura", "fel", "från", "får", "ganger", "gång", "gånger", "gör", "göra",
+ "hittar", "hjalp", "hjalpa", "hjälp", "hjälpa", "här", "installningarna", "installningen",
+ "inställningarna", "inställningen", "inte", "jag", "kommer", "kraschar", "kvittot", "losenord",
+ "lösenord", "mig", "min", "mitt", "måste", "nar", "när", "nästa", "och", "också", "om",
+ "oppnar", "paket", "ska", "skicka", "skulle", "sparningen", "spårningen", "säger", "till",
+ "tva", "två", "upp", "uppdaterats", "utan", "vi", "vill", "är", "återbetala", "återbetalning",
+ "öppnar",
+};
+
+inline constexpr const char* k_stop_8[] = {
+ "aceasta", "această", "acest", "acesta", "acum", "ale", "care", "dar", "din", "după", "este",
+ "foarte", "fost", "fără", "lui", "mi", "nu", "pentru", "până", "sunt", "să", "trebuie",
+ "vreau", "vă", "în", "și", "ți",
+};
+
+inline constexpr const char* k_stop_9[] = {
+ "abar", "ajke", "akhon", "amader", "amake", "amar", "ami", "amra", "apnake", "apnar", "apnara",
+ "apni", "ar", "asbe", "bhai", "bhalo", "bolte", "bolun", "chai", "chaina", "dhonnobad",
+ "dilam", "dite", "diye", "diyechi", "dorkar", "duibar", "ei", "eita", "ekbar", "ekhon",
+ "ekhono", "ekta", "ferot", "geche", "gese", "hobe", "hocche", "hoise", "hoye", "hoyeche",
+ "hoyni", "jabe", "jodi", "jonno", "kalke", "keno", "keu", "kharap", "khub", "ki", "kibhabe",
+ "kichu", "kintu", "kivabe", "kobe", "kokhon", "korbo", "korchi", "kore", "koreche", "korechi",
+ "koren", "korlam", "korsi", "korte", "korun", "kothay", "koto", "lagbe", "moddhe", "nai",
+ "niye", "oi", "oita", "onek", "ota", "pabo", "paini", "parben", "parbo", "parchi", "parchina",
+ "peyechi", "sathe", "shathe", "shob", "shomossa", "somossa", "tader", "tahole", "taka",
+ "theke", "tomake", "tomar", "tomra", "tumi", "valo",
+};
+
+inline constexpr const char* k_stop_10[] = {
+ "amma", "ancaq", "artiq", "artıq", "bir", "biz", "bu", "cox", "daha", "deyil", "də", "eger",
+ "görə", "həm", "hər", "ile", "ilə", "isə", "kimi", "lakin", "mən", "nə", "olan", "olmasa",
+ "olub", "onlar", "siz", "sonra", "sən", "ucun", "var", "ve", "və", "yalniz", "yalnız", "yox",
+ "yoxdur", "çox", "üçün", "əgər",
+};
+
+struct StopSpec { const char* code; const char* const* words; size_t count; };
+// In lang.py declaration order: the language that wins a tied score is the first listed.
+inline constexpr StopSpec k_stop_lists[] = {
+ {"en", k_stop_0, 116},
+ {"fr", k_stop_1, 67},
+ {"de", k_stop_2, 63},
+ {"es", k_stop_3, 65},
+ {"pt", k_stop_4, 107},
+ {"it", k_stop_5, 85},
+ {"nl", k_stop_6, 19},
+ {"sv", k_stop_7, 66},
+ {"ro", k_stop_8, 27},
+ {"bn", k_stop_9, 98},
+ {"az", k_stop_10, 40},
+};
+
+inline constexpr const char* k_short_swedish[] = {
+ "abonnemanget", "aterbetalning", "atkomst", "behover", "behöver", "betalningen", "fakturan",
+ "felmeddelande", "glomt", "glömt", "inloggningen", "installningen", "inställningen", "kvitto",
+ "kvittot", "losenord", "lösenord", "sparningen", "spårningen", "återbetalning", "åtkomst",
+};
+
+inline constexpr const char* k_shared[] = {
+ "as", "como", "con", "da", "das", "del", "dois", "dos", "e", "era", "esta", "este", "está",
+ "får", "hej", "il", "in", "is", "ja", "jo", "kommer", "la", "le", "les", "lo", "mais", "mi",
+ "mig", "min", "nada", "nej", "nos", "o", "om", "para", "por", "porque", "quando", "que", "se",
+ "ser", "skulle", "su", "sua", "tack", "tu", "un", "una", "vi", "was",
+};
+
+inline constexpr const char* k_nordic_overlap[] = {
+ "får", "hej", "ja", "jo", "kommer", "mig", "min", "nej", "om", "skulle", "tack", "vi",
+};
+
+inline constexpr const char* k_en_only[] = {
+ "after", "already", "always", "and", "any", "are", "aren", "at", "be", "because", "been",
+ "before", "being", "between", "both", "but", "can", "could", "couldn", "didn", "didnt", "does",
+ "doesn", "doesnt", "doing", "done", "dont", "down", "during", "each", "ever", "every", "few",
+ "for", "from", "has", "hasn", "have", "haven", "havent", "here", "him", "his", "how", "i",
+ "if", "into", "isn", "isnt", "it", "ive", "many", "might", "much", "must", "never", "nor",
+ "not", "of", "on", "only", "onto", "our", "please", "shall", "she", "should", "some", "than",
+ "that", "thats", "the", "their", "them", "then", "there", "these", "they", "this", "those",
+ "through", "to", "too", "until", "very", "wasn", "wasnt", "we", "were", "weren", "what",
+ "whats", "when", "where", "which", "while", "who", "whom", "why", "will", "with", "within",
+ "without", "won", "wont", "would", "wouldn", "yet", "you", "your", "youre", "yours",
+};
+
+inline constexpr const char* k_en_collision[] = {
+ "care", "come", "do", "im", "per", "plus", "son", "todo",
+};
+
+inline constexpr uint32_t k_non_en_diacritics[] = {
+ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 241, 242,
+ 243, 244, 245, 246, 248, 249, 250, 251, 252, 253, 255, 257, 259, 261, 263, 269, 271, 273, 275,
+ 281, 283, 287, 291, 299, 305, 311, 316, 322, 324, 326, 328, 337, 339, 345, 347, 351, 353, 355,
+ 357, 363, 367, 369, 378, 380, 382, 537, 539, 601,
+};
+
+inline constexpr double k_non_en_diacritic_rate = 0.02;
+inline constexpr double k_english_rescue_diacritic_rate = 0.06;
+inline constexpr double k_non_latin_fraction = 0.2;
+inline constexpr double k_non_latin_min_fraction = 0.1;
+inline constexpr int k_non_latin_min_letters = 10;
diff --git a/laya-cpp/tests/lang_test.cpp b/laya-cpp/tests/lang_test.cpp
new file mode 100644
index 00000000..9842dcd6
--- /dev/null
+++ b/laya-cpp/tests/lang_test.cpp
@@ -0,0 +1,125 @@
+#include <cmath>
+
+#include "golden.hpp"
+#include "laya/lang.hpp"
+#include "laya/sha256.hpp"
+#include "laya/unicodedata.hpp"
+
+using laya_test::At;
+using laya_test::Value;
+namespace lg = laya::lang;
+namespace pv = laya::pyvalue;
+
+namespace {
+// The state a golden row describes.
+pv::Value StateOf(const Value& enc) {
+ const std::string& kind = enc.Find("kind")->as_string();
+ const Value& v = *enc.Find("value");
+ if (kind == "bytes") {
+ std::string b;
+ for (size_t i = 0; i < v.size(); i++) b.push_back(char(v.value(i).as_int()));
+ return laya::unicodedata::ValidUtf8(b) ? pv::Value::String(b) : pv::Value::Null(); // undecodable bytes read as nothing
+ }
+ return v;
+}
+void ExpectProfile(const Value& want, const lg::Profile& got, const std::string& what) {
+ ASSERT_EQ(want.size(), got.size()) << what;
+ for (size_t i = 0; i < got.size(); i++) {
+ EXPECT_EQ(want.key(i), got[i].first) << what;
+ EXPECT_EQ(laya_test::Num(want.value(i)), got[i].second) << what << " " << got[i].first;
+ }
+}
+} // namespace
+
+class DetectRow : public ::testing::TestWithParam<int> {};
+TEST_P(DetectRow, MatchesPython) {
+ LAYA_GOLDEN_LIST(rows, "lang_detect.json", "cases");
+ const Value& row = rows->value(size_t(GetParam()));
+ const std::string name = row.Find("name")->as_string();
+ pv::Value state = StateOf(*row.Find("state"));
+ const std::string text = row.Find("state_text")->as_string();
+ EXPECT_EQ(lg::StateText(state), text) << name;
+ EXPECT_EQ(lg::DetectScript(text), row.Find("detect_script")->as_string()) << name;
+ ExpectProfile(*row.Find("script_profile"), lg::ScriptProfile(text), name);
+ const Value& lp = *row.Find("latin_profile");
+ lg::LatinProfile got = lg::LatinProfileOf(text);
+ auto wl = lp.Find("language");
+ EXPECT_EQ(got.language, wl->is_null() ? "" : wl->as_string()) << name;
+ EXPECT_EQ(got.english_hits, lp.Find("english_hits")->as_int()) << name;
+ EXPECT_EQ(got.diacritic_rate, laya_test::Num(*lp.Find("diacritic_rate"))) << name;
+ EXPECT_EQ(got.looks_non_english, lp.Find("looks_non_english")->as_bool()) << name;
+ const Value& an = *row.Find("analyse");
+ lg::Analysis a = lg::Analyze(state);
+ EXPECT_EQ(a.script, an.Find("script")->as_string()) << name;
+ ExpectProfile(*an.Find("script_profile"), a.script_profile, name);
+ auto al = an.Find("language");
+ EXPECT_EQ(a.language, al->is_null() ? "" : al->as_string()) << name;
+ EXPECT_EQ(a.english, an.Find("is_english")->as_bool()) << name;
+ EXPECT_EQ(a.diacritic_rate, laya_test::Num(*an.Find("diacritic_rate"))) << name;
+ EXPECT_EQ(a.non_latin_fraction, laya_test::Num(*an.Find("non_latin_fraction"))) << name;
+ auto ms = an.Find("mixed_segment");
+ EXPECT_EQ(a.mixed_segment, ms->is_null() ? "" : ms->as_string()) << name;
+ EXPECT_EQ(lg::IsEnglish(state), row.Find("is_english")->as_bool()) << name;
+ EXPECT_EQ(lg::DetectLanguage(state), a.language) << name;
+}
+INSTANTIATE_TEST_SUITE_P(Golden, DetectRow, ::testing::ValuesIn(laya_test::Rows("lang_detect.json", {"cases"})));
+
+TEST(Lang, ScriptOfMatchesPythonEverywhere) {
+ LAYA_GOLDEN_LIST(want, "lang_detect.json", "unicode_digests", "script_of");
+ laya::Sha256 h;
+ for (char32_t r = 0; r < 0x110000; r++) h.Update(lg::ScriptOf(r) + ";");
+ EXPECT_EQ(laya::ToHex(h.Finish()), want->as_string());
+}
+
+TEST(Lang, StateShapesMatchPython) {
+ LAYA_GOLDEN_LIST(shapes, "go_extra.json", "state_shapes");
+ const std::string german = shapes->Find("string")->Find("state_text")->as_string();
+ auto S = [](const std::string& s) { return pv::Value::String(s); };
+ auto L = [](std::vector<pv::Value> v) { return pv::Value::Array(std::move(v)); };
+ auto O = [](const char* k, pv::Value v) {
+ pv::Value o = pv::Value::Object();
+ o.Set(k, std::move(v));
+ return o;
+ };
+ pv::Value seven = S(german), six = S(german);
+ for (int i = 0; i < 7; i++) seven = L({seven});
+ for (int i = 0; i < 6; i++) six = L({six});
+ std::vector<std::pair<std::string, pv::Value>> built = {
+ {"string", S(german)},
+ {"map_of_slices", O("a", L({S(german)}))},
+ {"slice_of_maps", L({O("a", S(german))})},
+ {"map_of_objects", O("a", O("b", S(german)))},
+ {"slice_of_slices", L({L({S(german)})})},
+ {"tuple", L({S(german)})},
+ {"seven_deep", seven},
+ {"six_deep", six},
+ {"numbers_and_none", L({pv::Value::Int(1), pv::Value::Float(2.5), pv::Value::Null(), pv::Value::Bool(true)})},
+ {"mixed", O("k", L({S(german), pv::Value::Int(3), O("x", pv::Value::Null())}))},
+ {"bytes", S(german)},
+ {"invalid_bytes", pv::Value::Null()}};
+ for (const auto& b : built) {
+ const Value* want = shapes->Find(b.first);
+ ASSERT_NE(want, nullptr) << b.first;
+ lg::Analysis a = lg::Analyze(b.second);
+ EXPECT_EQ(lg::StateText(b.second), want->Find("state_text")->as_string()) << b.first;
+ EXPECT_EQ(a.english, want->Find("is_english")->as_bool()) << b.first;
+ auto wl = want->Find("language");
+ EXPECT_EQ(a.language, wl->is_null() ? "" : wl->as_string()) << b.first;
+ EXPECT_EQ(a.script, want->Find("script")->as_string()) << b.first;
+ }
+}
+
+TEST(Lang, InvalidUtf8MatchesPython) {
+ LAYA_GOLDEN_LIST(fam, "go_extra.json", "invalid_utf8");
+ const Value& hex = *fam->Find("hex");
+ const Value& script = *fam->Find("script");
+ const Value& text = *fam->Find("state_text");
+ ASSERT_EQ(hex.size(), 3000u);
+ for (size_t i = 0; i < hex.size(); i++) {
+ const std::string& h = hex.value(i).as_string();
+ std::string bytes;
+ for (size_t k = 0; k + 1 < h.size(); k += 2) bytes.push_back(char(std::stoi(h.substr(k, 2), nullptr, 16)));
+ EXPECT_EQ(lg::DetectScript(bytes), script.value(i).as_string()) << "row " << i;
+ EXPECT_EQ(lg::StateText(pv::Value::String(bytes)), text.value(i).as_string()) << "row " << i;
+ }
+}
diff --git a/laya-go/scripts/gen_tables.py b/laya-go/scripts/gen_tables.py
index e05a4c59..9aa3a986 100644
--- a/laya-go/scripts/gen_tables.py
+++ b/laya-go/scripts/gen_tables.py
@@ -36,6 +36,7 @@ OUT_UNICODE = os.path.join(GO, "internal", "unicodedata", "tables_gen.go")
OUT_LANG = os.path.join(GO, "lang", "tables_gen.go")
OUT_TOKENIZER = os.path.join(GO, "tokenizer", "tables_gen.go")
OUT_CPP_UNICODE = os.path.join(REPO, "laya-cpp", "src", "unicodedata_tables.inc")
+OUT_CPP_LANG = os.path.join(REPO, "laya-cpp", "src", "lang_tables.inc")
def wrap(values, indent="\t", width=100):
@@ -280,6 +281,49 @@ def render_lang():
return "".join(out)
+def render_lang_cpp():
+ """laya.lang's script ranges, function words and thresholds, as a C++ include for src/lang.cpp."""
+ sys.path.insert(0, REPO)
+ from laya import lang
+
+ flat = sorted((lo, hi, name) for name, rs in lang._SCRIPT_RANGES for lo, hi in rs)
+ for (a_lo, a_hi, a_name), (b_lo, b_hi, b_name) in zip(flat, flat[1:]):
+ if a_hi >= b_lo:
+ raise SystemExit("gen_tables: script ranges %s and %s overlap" % (a_name, b_name))
+ names = [name for name, _ in lang._SCRIPT_RANGES]
+ index_of = {name: n for n, name in enumerate(names)}
+
+ def words(name, ws):
+ return "inline constexpr const char* %s[] = {\n%s\n};\n\n" % (
+ name, wrap([go_str(w) for w in sorted(ws)], indent=" "))
+
+ out = ["// Code generated by laya-go/scripts/gen_tables.py --lang cpp from laya/lang.py; DO NOT EDIT.\n"
+ "//\n// Regenerate with: python laya-go/scripts/gen_tables.py --lang cpp\n\n"]
+ out.append("inline constexpr const char* k_script_names[] = {%s};\n\n" % ", ".join(go_str(n) for n in names))
+ out.append(cpp_array("k_script_lookup", [v for lo, hi, _ in flat for v in (lo, hi)]))
+ out.append(cpp_array("k_script_lookup_index", [index_of[n] for _, _, n in flat], "uint8_t"))
+ out.append("inline constexpr uint32_t k_script_max_code_point = %#x;\n\n" % max(hi for _, hi, _ in flat))
+ stops = list(lang._STOP.items())
+ for n, (code, ws) in enumerate(stops):
+ out.append(words("k_stop_%d" % n, ws))
+ out.append("struct StopSpec { const char* code; const char* const* words; size_t count; };\n"
+ "// In lang.py declaration order: the language that wins a tied score is the first listed.\n"
+ "inline constexpr StopSpec k_stop_lists[] = {\n%s\n};\n\n"
+ % "\n".join(" {%s, k_stop_%d, %d}," % (go_str(c), n, len(ws)) for n, (c, ws) in enumerate(stops)))
+ for name, ws in (("k_short_swedish", lang._SHORT_SWEDISH_WORDS), ("k_shared", lang._SHARED_WORDS),
+ ("k_nordic_overlap", lang._NORDIC_OVERLAP_WORDS), ("k_en_only", lang._EN_ONLY_WORDS),
+ ("k_en_collision", lang._EN_COLLISION_WORDS)):
+ out.append(words(name, ws))
+ out.append(cpp_array("k_non_en_diacritics", sorted(ord(c) for c in lang._NON_EN_DIACRITICS)))
+ for name, value in (("k_non_en_diacritic_rate", lang.NON_EN_DIACRITIC_RATE),
+ ("k_english_rescue_diacritic_rate", lang.ENGLISH_RESCUE_DIACRITIC_RATE),
+ ("k_non_latin_fraction", lang.NON_LATIN_FRACTION),
+ ("k_non_latin_min_fraction", lang.NON_LATIN_MIN_FRACTION)):
+ out.append("inline constexpr double %s = %r;\n" % (name, value))
+ out.append("inline constexpr int k_non_latin_min_letters = %d;\n" % lang.NON_LATIN_MIN_LETTERS)
+ return "".join(out)
+
+
# ---------------------------------------------------------------------------------------------
# The tokenizers crate: pre-tokenizer classes, single_word flank, lowercase, normalization.
# ---------------------------------------------------------------------------------------------
@@ -459,8 +503,8 @@ def render_normalization(normalizers):
TARGETS = ((OUT_UNICODE, render_unicode), (OUT_LANG, render_lang), (OUT_TOKENIZER, render_tokenizer))
-# laya-cpp: the lang tables join in C1 part 2, the tokenizer tables in C2.
-CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)),)
+# laya-cpp: the tokenizer tables join in C2.
+CPP_TARGETS = ((OUT_CPP_UNICODE, lambda: render_unicode(cpp=True)), (OUT_CPP_LANG, render_lang_cpp))
def main(argv=None):