Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(dotnet): add language detection

kevin-gatimu8de1e1a0b56e
diff --git a/laya-dotnet/src/Laya/LanguageDetection.cs b/laya-dotnet/src/Laya/LanguageDetection.cs
new file mode 100644
index 00000000..73ef0b2c
--- /dev/null
+++ b/laya-dotnet/src/Laya/LanguageDetection.cs
@@ -0,0 +1,525 @@
+using System.Globalization;
+using System.Text;
+
+namespace Laya;
+
+/// <summary>
+/// Dependency-free script and Latin-language detection used to route between Laya checkpoints.
+/// Ports <c>laya/lang.py</c>.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Routing only needs one decision: is this English Latin text, or is it something the English
+/// checkpoint cannot read? Benchmarks on MASSIVE (14 languages) showed the English checkpoint
+/// collapsing to near-random on non-Latin scripts, while holding up far better on Latin-script
+/// languages. So the signal that matters most is script, and the secondary signal is whether
+/// Latin text is English.
+/// </para>
+/// <para>
+/// Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic
+/// and is explicitly best-effort: pass an explicit model or language when it is already known.
+/// </para>
+/// <para>
+/// Every method here iterates Unicode <em>code points</em> (via <see cref="Rune"/>), never UTF-16
+/// code units, and applies Unicode-category checks rather than ASCII-only ones — matching
+/// Python's <c>str.isalpha()</c>, <c>len()</c>, slicing and <c>str.lower()</c> semantics exactly.
+/// A UTF-16-unit-based port would split astral characters (emoji, Deseret, Mathematical
+/// Alphanumeric Symbols) into two units and disagree with Python on script counts and truncation
+/// boundaries.
+/// </para>
+/// </remarks>
+public static class LanguageDetection
+{
+ // Unicode blocks the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
+ // Order matters: it decides which non-Latin script wins a tie in DetectScript (first
+ // encountered in the source text wins, and Latin never wins a tie against a script that is
+ // actually present, because "latin" is appended to the tally only after the scan finishes).
+ private static readonly (string Name, (int Lo, int Hi)[] Ranges)[] ScriptRanges =
+ [
+ ("greek", [(0x0370, 0x03FF), (0x1F00, 0x1FFF)]),
+ ("cyrillic", [(0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F)]),
+ ("armenian", [(0x0530, 0x058F)]),
+ ("hebrew", [(0x0590, 0x05FF)]),
+ ("arabic", [(0x0600, 0x06FF), (0x0750, 0x077F), (0x08A0, 0x08FF), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF)]),
+ ("devanagari", [(0x0900, 0x097F), (0xA8E0, 0xA8FF)]),
+ ("bengali", [(0x0980, 0x09FF)]),
+ ("gurmukhi", [(0x0A00, 0x0A7F)]),
+ ("gujarati", [(0x0A80, 0x0AFF)]),
+ ("oriya", [(0x0B00, 0x0B7F)]),
+ ("tamil", [(0x0B80, 0x0BFF)]),
+ ("telugu", [(0x0C00, 0x0C7F)]),
+ ("kannada", [(0x0C80, 0x0CFF)]),
+ ("malayalam", [(0x0D00, 0x0D7F)]),
+ ("sinhala", [(0x0D80, 0x0DFF)]),
+ ("thai", [(0x0E00, 0x0E7F)]),
+ ("lao", [(0x0E80, 0x0EFF)]),
+ ("tibetan", [(0x0F00, 0x0FFF)]),
+ ("myanmar", [(0x1000, 0x109F)]),
+ ("georgian", [(0x10A0, 0x10FF)]),
+ ("ethiopic", [(0x1200, 0x137F)]),
+ ("khmer", [(0x1780, 0x17FF)]),
+ ("hangul", [(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)]),
+ ("kana", [(0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF)]),
+ ("han", [(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF)]),
+ ];
+
+ // Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is
+ // weighted and a margin is required before calling something non-English. Order is load
+ // bearing: LatinProfile's tie-break walks this list (excluding "en") and the first language
+ // with the highest score wins.
+ private static readonly string[] StopOrder = ["en", "fr", "de", "es", "pt", "it", "nl", "ro"];
+
+ private static readonly Dictionary<string, HashSet<string>> Stop = new(StringComparer.Ordinal)
+ {
+ ["en"] = new(StringComparer.Ordinal)
+ {
+ "the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
+ "this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
+ "will", "can", "would", "there", "their", "what", "which", "please", "we", "i",
+ },
+ ["fr"] = new(StringComparer.Ordinal)
+ {
+ "le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
+ "pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce",
+ "et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",
+ "ma", "ta", "sa", "mes", "tes", "ses", "ces", "deux", "trois", "très", "bien",
+ "tout", "tous", "toute", "fait", "veux", "veut", "peux", "peut", "dois", "doit",
+ "merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi",
+ "alors", "donc",
+ },
+ ["de"] = new(StringComparer.Ordinal)
+ {
+ "der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
+ "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",
+ },
+ ["es"] = new(StringComparer.Ordinal)
+ {
+ "el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
+ "pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",
+ "la", "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos",
+ "ni", "dos", "tres", "fue", "fueron", "ser", "tiene", "tienen", "tengo", "puede",
+ "pueden", "quiero", "necesito", "hemos", "han", "sobre", "entre", "cuando", "donde",
+ "porque", "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada", "algo",
+ "aquí", "hoy", "gracias",
+ },
+ ["pt"] = new(StringComparer.Ordinal)
+ {
+ "os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
+ "dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela",
+ "o", "e", "na", "nas", "nos", "ao", "aos", "por", "foi", "era", "ser", "sou",
+ "tem", "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu",
+ "sua", "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais",
+ "já", "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
+ "olá",
+ },
+ ["it"] = new(StringComparer.Ordinal)
+ {
+ "il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
+ "questo", "questa", "anche", "come", "più", "nella", "alla",
+ "la", "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi",
+ "ci", "ne", "ho", "hai", "ha", "abbiamo", "avete", "hanno", "era", "stato", "stata",
+ "devo", "deve", "devono", "voglio", "vorrei", "mio", "mia", "tuo", "sua", "quando",
+ "dove", "perche", "molto", "poco", "sempre", "mai", "già", "ancora", "adesso", "oggi",
+ "ieri", "grazie", "ciao", "scusa",
+ "nel", "nell", "negli", "sul", "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei",
+ "delle", "dello", "degli", "agli", "alle", "col",
+ },
+ ["nl"] = new(StringComparer.Ordinal)
+ {
+ "het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
+ "door", "maar", "ook", "worden", "deze", "naar", "wordt",
+ },
+ ["ro"] = new(StringComparer.Ordinal)
+ {
+ "și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
+ "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
+ "acesta", "aceasta", "mi", "ți", "vă", "nu",
+ },
+ };
+
+ // Letters that ordinary English does not use. This is the signal that catches a Latin-script
+ // language for which no stopword list is held at all (Romanian, Polish, Czech, Turkish,
+ // Baltic, ...).
+ private const string NonEnDiacriticChars =
+ "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" // Western European
+ + "ăâîșțşţ" // Romanian
+ + "ąćęłńśźż" // Polish
+ + "čďěňřšťůž" // Czech / Slovak
+ + "őű" // Hungarian
+ + "ğı" // Turkish (text is lowercased before matching)
+ + "āēģīķļņūž" // Baltic
+ + "đ"; // Serbo-Croatian / Vietnamese
+
+ private static readonly HashSet<int> NonEnDiacritics =
+ new(NonEnDiacriticChars.EnumerateRunes().Select(r => r.Value));
+
+ // Words that more than one list claims. Matching one says "not English" without saying
+ // *which* language, so it alone may not name a winner (see LatinProfile).
+ private static readonly HashSet<string> SharedWords = ComputeSharedWords();
+
+ /// <summary>
+ /// A diacritic rate above this is taken as evidence the text is not English, even when no
+ /// stopword list matches it.
+ /// </summary>
+ private const double NonEnDiacriticRate = 0.02;
+
+ private static HashSet<string> ComputeSharedWords()
+ {
+ var counts = new Dictionary<string, int>(StringComparer.Ordinal);
+ foreach (var set in Stop.Values)
+ foreach (var w in set)
+ counts[w] = counts.GetValueOrDefault(w) + 1;
+ return new HashSet<string>(counts.Where(kv => kv.Value > 1).Select(kv => kv.Key), StringComparer.Ordinal);
+ }
+
+ /// <summary>
+ /// Flatten a state into the text used for detection: string leaves of a string, dict-like or
+ /// list-like value, joined with a single space and truncated to <paramref name="maxChars"/>
+ /// Unicode code points. Keys are ignored, since they are usually English.
+ /// </summary>
+ /// <remarks>Recursion stops past 6 levels of nesting, matching Python's <c>_iter_text</c>.</remarks>
+ public static string StateText(object? state, int maxChars = 4000)
+ {
+ var leaves = new List<string>();
+ CollectText(state, leaves, 0);
+ return SliceCodePoints(string.Join(" ", leaves), maxChars);
+ }
+
+ private static void CollectText(object? state, List<string> leaves, int depth)
+ {
+ if (depth > 6 || state is null) return;
+ switch (state)
+ {
+ case string s:
+ leaves.Add(s);
+ return;
+ // Dict-like before generically enumerable, mirroring PythonJson's dispatch order.
+ case IEnumerable<KeyValuePair<string, object?>> kvpSeq:
+ foreach (var kv in kvpSeq) CollectText(kv.Value, leaves, depth + 1);
+ return;
+ case System.Collections.IDictionary dict:
+ foreach (System.Collections.DictionaryEntry entry in dict) CollectText(entry.Value, leaves, depth + 1);
+ return;
+ case System.Collections.IEnumerable seq:
+ foreach (var item in seq) CollectText(item, leaves, depth + 1);
+ return;
+ default:
+ // Numbers, booleans and other scalars contribute nothing, matching _iter_text.
+ return;
+ }
+ }
+
+ /// <summary>
+ /// Dominant script of <paramref name="text"/>: <c>"latin"</c>, <c>"han"</c>,
+ /// <c>"devanagari"</c>, ... or <c>"unknown"</c> if there are no letters.
+ /// </summary>
+ public static string DetectScript(string text)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+
+ // Non-Latin scripts are tallied in the order first encountered in the text; "latin" is
+ // appended only after the scan, so it never wins a tie against a script actually present,
+ // regardless of where in the text the Latin characters fall.
+ var order = new List<string>();
+ var counts = new Dictionary<string, int>(StringComparer.Ordinal);
+ var latin = 0;
+
+ foreach (var rune in text.EnumerateRunes())
+ {
+ if (!Rune.IsLetter(rune)) continue;
+ var cp = rune.Value;
+ if (cp < 0x0250 || (cp >= 0x1E00 && cp <= 0x1EFF)) { latin++; continue; }
+ foreach (var (name, ranges) in ScriptRanges)
+ {
+ if (!InRanges(cp, ranges)) continue;
+ if (!counts.ContainsKey(name)) order.Add(name);
+ counts[name] = counts.GetValueOrDefault(name) + 1;
+ break;
+ }
+ }
+
+ counts["latin"] = latin;
+ order.Add("latin");
+
+ var total = counts.Values.Sum();
+ if (total == 0) return "unknown";
+
+ var bestName = order[0];
+ var bestCount = counts[bestName];
+ for (var i = 1; i < order.Count; i++)
+ {
+ var name = order[i];
+ if (counts[name] > bestCount) { bestName = name; bestCount = counts[name]; }
+ }
+ return bestName;
+ }
+
+ /// <summary>Fraction of alphabetic characters belonging to each detected script.</summary>
+ public static IReadOnlyDictionary<string, double> ScriptProfile(string text)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+
+ var counts = new Dictionary<string, int>(StringComparer.Ordinal) { ["latin"] = 0 };
+ foreach (var rune in text.EnumerateRunes())
+ {
+ if (!Rune.IsLetter(rune)) continue;
+ var cp = rune.Value;
+ if (cp < 0x0250 || (cp >= 0x1E00 && cp <= 0x1EFF)) { counts["latin"]++; continue; }
+ foreach (var (name, ranges) in ScriptRanges)
+ {
+ if (!InRanges(cp, ranges)) continue;
+ counts[name] = counts.GetValueOrDefault(name) + 1;
+ break;
+ }
+ }
+
+ var total = counts.Values.Sum();
+ if (total == 0) return new Dictionary<string, double>(0);
+
+ var result = new Dictionary<string, double>(StringComparer.Ordinal);
+ foreach (var (k, v) in counts)
+ if (v != 0) result[k] = (double)v / total;
+ return result;
+ }
+
+ private static bool InRanges(int cp, (int Lo, int Hi)[] ranges)
+ {
+ foreach (var (lo, hi) in ranges)
+ if (cp >= lo && cp <= hi) return true;
+ return false;
+ }
+
+ /// <summary>
+ /// Evidence behind the Latin-script language guess: <see cref="Language"/> (may be
+ /// <see langword="null"/> when undecided), <see cref="EnglishHits"/>,
+ /// <see cref="DiacriticRate"/> and <see cref="LooksNonEnglish"/>. <see cref="LanguageDetection.Analyse"/>
+ /// needs the evidence and not just the verdict, because "undecided" and "English" are
+ /// different answers and only one of them is safe to send to the English checkpoint.
+ /// </summary>
+ /// <param name="Language">
+ /// The best-effort language code, or <see langword="null"/> when undecided. Only named when
+ /// it matched at least one word that no other list claims: shared function words alone
+ /// (<c>la</c>, <c>e</c>, <c>o</c>) identify no particular language.
+ /// </param>
+ /// <param name="EnglishHits">How many words matched the English stopword list.</param>
+ /// <param name="DiacriticRate">
+ /// Fraction of (lowercased) code points that are letters ordinary English does not use.
+ /// Not rounded; <see cref="LanguageAnalysis.DiacriticRate"/> rounds this to 4 decimals.
+ /// </param>
+ /// <param name="LooksNonEnglish"><see cref="DiacriticRate"/> at or above the detection threshold.</param>
+ public readonly record struct LatinProfileResult(
+ string? Language, int EnglishHits, double DiacriticRate, bool LooksNonEnglish);
+
+ /// <summary>
+ /// Evidence behind the Latin-script language guess. See <see cref="LatinProfileResult"/>.
+ /// </summary>
+ public static LatinProfileResult LatinProfile(string text)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+
+ var words = ExtractWords(text).Select(PythonLower).ToList();
+ var lowered = PythonLower(text);
+
+ var diac = 0;
+ var loweredLen = 0;
+ foreach (var rune in lowered.EnumerateRunes())
+ {
+ loweredLen++;
+ if (NonEnDiacritics.Contains(rune.Value)) diac++;
+ }
+ var diacRate = (double)diac / Math.Max(1, loweredLen);
+ var nonEnglish = diacRate >= NonEnDiacriticRate;
+
+ if (words.Count < 4)
+ return new LatinProfileResult(null, 0, diacRate, nonEnglish);
+
+ var wordSet = new HashSet<string>(words, StringComparer.Ordinal);
+ var scores = new Dictionary<string, int>(StringComparer.Ordinal);
+ foreach (var lang in StopOrder)
+ scores[lang] = words.Count(w => Stop[lang].Contains(w));
+ var en = scores.GetValueOrDefault("en");
+
+ // Only a language that matched at least one word no other list claims may be named.
+ // Without that condition the top score can be pure overlap. Such a language is dropped
+ // entirely from the running rather than merely losing the tie, so a lesser score with
+ // real evidence still gets named, and the text stays undecided when no list has any.
+ string? bestLg = null;
+ var best = 0;
+ foreach (var lang in StopOrder)
+ {
+ if (lang == "en") continue;
+ var evidenced = false;
+ foreach (var w in wordSet)
+ if (Stop[lang].Contains(w) && !SharedWords.Contains(w)) { evidenced = true; break; }
+ if (!evidenced) continue;
+
+ var score = scores[lang];
+ if (bestLg is null || score > best) { bestLg = lang; best = score; }
+ }
+
+ string? language = null;
+ if (bestLg is not null && best >= Math.Max(2, en + 2))
+ {
+ // a non-English language needs a clear margin over English function words
+ language = bestLg;
+ }
+ else if (bestLg is not null && nonEnglish && best >= Math.Max(2, en))
+ {
+ // Needs two hits here too: one shared function word on the strength of the
+ // diacritics alone is a guess dressed as a detection.
+ language = bestLg;
+ }
+ else if (en > 0 && !nonEnglish)
+ {
+ language = "en";
+ }
+
+ return new LatinProfileResult(language, en, diacRate, nonEnglish);
+ }
+
+ /// <summary>
+ /// Best-effort language code for Latin-script text, or <see langword="null"/> when
+ /// undecided.
+ /// </summary>
+ /// <remarks>
+ /// Scores function-word hits per language and requires the winner to beat English by a
+ /// margin and to have matched at least one word of its own, so ordinary English is never
+ /// misrouted and a word of several languages at once names none of them. Short inputs
+ /// usually return <see langword="null"/> on purpose.
+ /// </remarks>
+ public static string? GuessLatinLanguage(string text) => LatinProfile(text).Language;
+
+ /// <summary>Full detection result for a state.</summary>
+ public static LanguageAnalysis Analyse(object? state)
+ {
+ var text = StateText(state);
+ var prof = ScriptProfile(text);
+ var script = DetectScript(text);
+ var nonLatin = prof.Count > 0 ? Calibration.Round4(1.0 - prof.GetValueOrDefault("latin", 0.0)) : 0.0;
+
+ if (script == "unknown")
+ return new LanguageAnalysis("unknown", prof, null, true, true, 0.0, 0.0);
+
+ if (script != "latin")
+ return new LanguageAnalysis(script, prof, null, false, true, 0.0, nonLatin);
+
+ var profLat = LatinProfile(text);
+ var lang = profLat.Language;
+ // Undecided is not English. Treating it as English sends every Latin-script language no
+ // stopwords are held for to the checkpoint that cannot read it, silently.
+ var undecided = lang is null;
+ var english = lang == "en" || (undecided && !profLat.LooksNonEnglish);
+ return new LanguageAnalysis("latin", prof, lang, english, undecided,
+ Calibration.Round4(profLat.DiacriticRate), nonLatin);
+ }
+
+ /// <summary>Whether the English checkpoint can be expected to read this state.</summary>
+ public static bool IsEnglish(object? state) => Analyse(state).IsEnglish;
+
+ // ── word extraction ──────────────────────────────────────────────────────
+
+ /// <summary>
+ /// Runs of code points matching Python's <c>[^\W\d_]+</c>: word characters (letters, plus
+ /// non-decimal numeric characters such as Roman numerals) excluding decimal digits and the
+ /// underscore.
+ /// </summary>
+ private static List<string> ExtractWords(string text)
+ {
+ var words = new List<string>();
+ var current = new StringBuilder();
+ foreach (var rune in text.EnumerateRunes())
+ {
+ if (IsWordRune(rune))
+ {
+ current.Append(rune);
+ }
+ else if (current.Length > 0)
+ {
+ words.Add(current.ToString());
+ current.Clear();
+ }
+ }
+ if (current.Length > 0) words.Add(current.ToString());
+ return words;
+ }
+
+ private static bool IsWordRune(Rune r)
+ {
+ if (Rune.IsLetter(r)) return true;
+ var category = Rune.GetUnicodeCategory(r);
+ // Python's \w = str.isalnum() (letters, plus Nd/Nl/No numeric categories) or "_"; \d
+ // matches only decimal digits (Nd). [^\W\d_]+ therefore keeps letters and the
+ // non-decimal numeric categories (Nl, No — e.g. Roman numerals, vulgar fractions) but
+ // excludes Nd and the literal underscore.
+ return category is UnicodeCategory.LetterNumber or UnicodeCategory.OtherNumber;
+ }
+
+ // ── code-point-aware slicing ─────────────────────────────────────────────
+
+ /// <summary>
+ /// Python's <c>str.lower()</c>: full (locale-independent default) Unicode lowercasing,
+ /// including the one unconditional 1-to-2-code-point special case, U+0130 (LATIN CAPITAL
+ /// LETTER I WITH DOT ABOVE) → U+0069 U+0307 ("i" + combining dot above).
+ /// </summary>
+ /// <remarks>
+ /// This project builds with <c>InvariantGlobalization</c> enabled, under which
+ /// <see cref="string.ToLowerInvariant()"/> uses a compiled-in simple (1-to-1) Unicode casing
+ /// table rather than ICU's full <c>SpecialCasing.txt</c> rules: it lowercases ordinary
+ /// accented letters correctly (é, ß, ...) but leaves U+0130 completely untouched rather than
+ /// expanding it, so the fix-up below only ever needs to look for a leftover U+0130.
+ /// </remarks>
+ internal static string PythonLower(string text)
+ {
+ var lowered = text.ToLowerInvariant();
+ return lowered.Contains('İ') ? lowered.Replace("İ", "i̇") : lowered;
+ }
+
+ /// <summary>Python's <c>s[:maxCodePoints]</c>: a prefix measured in Unicode code points.</summary>
+ /// <remarks>Internal rather than private so <see cref="LayaEmail"/> can reuse it for <c>max_chars</c>.</remarks>
+ internal static string SliceCodePoints(string text, int maxCodePoints)
+ {
+ if (maxCodePoints <= 0) return string.Empty;
+ var sb = new StringBuilder();
+ var count = 0;
+ foreach (var rune in text.EnumerateRunes())
+ {
+ if (count >= maxCodePoints) break;
+ sb.Append(rune);
+ count++;
+ }
+ return sb.ToString();
+ }
+}
+
+/// <summary>
+/// Full language/script detection result for a state. Every field is populated on every branch
+/// (script <c>"unknown"</c>, non-Latin, or Latin), so a caller never needs to guard on which
+/// branch produced it.
+/// </summary>
+/// <param name="Script">
+/// <c>"unknown"</c> (no letters), <c>"latin"</c>, or a non-Latin script name such as
+/// <c>"devanagari"</c> or <c>"han"</c>.
+/// </param>
+/// <param name="ScriptProfile">Fraction of alphabetic characters belonging to each detected script.</param>
+/// <param name="Language">Best-effort ISO-ish language code for Latin-script text, or <see langword="null"/>.</param>
+/// <param name="IsEnglish">Whether the English checkpoint can be expected to read this state.</param>
+/// <param name="LanguageUndecided">
+/// True when the script is not Latin (language is not applicable), or the script is Latin but no
+/// language could be identified.
+/// </param>
+/// <param name="DiacriticRate">
+/// Fraction of lowercased code points that are letters ordinary English does not use. Rounded to
+/// 4 decimals. Always 0 outside the Latin-script branch.
+/// </param>
+/// <param name="NonLatinFraction">
+/// <c>1 - (fraction of alphabetic characters that are Latin)</c>, rounded to 4 decimals; 0 when
+/// there are no letters at all.
+/// </param>
+public sealed record LanguageAnalysis(
+ string Script,
+ IReadOnlyDictionary<string, double> ScriptProfile,
+ string? Language,
+ bool IsEnglish,
+ bool LanguageUndecided,
+ double DiacriticRate,
+ double NonLatinFraction);
diff --git a/laya-dotnet/tests/Laya.Tests/LanguageDetectionTests.cs b/laya-dotnet/tests/Laya.Tests/LanguageDetectionTests.cs
new file mode 100644
index 00000000..a3cd4ca7
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/LanguageDetectionTests.cs
@@ -0,0 +1,262 @@
+using System.Text.Json;
+
+namespace Laya.Tests;
+
+/// <summary>
+/// Tier 1: script and Latin-language detection, which needs no artifacts. Named cases mirror
+/// <c>tests/test_router.py</c>'s language-detection sections; <see cref="EveryLangProbeEntry"/>
+/// consumes every entry of <c>golden/routing/lang_probe.json</c>.
+/// </summary>
+public sealed class LanguageDetectionTests
+{
+ // ── script detection ─────────────────────────────────────────────────────
+
+ [Theory]
+ [InlineData("The customer was charged twice and wants a refund.", "latin")]
+ [InlineData("Հայերեն", "armenian")]
+ [InlineData("ՀԱՅԵՐԵՆ", "armenian")]
+ [InlineData("։֊", "unknown")] // Armenian punctuation only: no letters
+ [InlineData("Le client a été facturé deux fois et demande un remboursement.", "latin")]
+ [InlineData("ग्राहक से दो बार शुल्क लिया गया और वह धनवापसी चाहता है।", "devanagari")]
+ [InlineData("お客様は二重に請求されたため返金を希望しています。", "kana")]
+ [InlineData("客户被重复扣款要求退款", "han")]
+ [InlineData("고객이 두 번 청구되어 환불을 원합니다", "hangul")]
+ [InlineData("تم خصم المبلغ مرتين من العميل ويريد استرداد الأموال", "arabic")]
+ [InlineData("வாடிக்கையாளரிடம் இருமுறை கட்டணம் வசூலிக்கப்பட்டது", "tamil")]
+ [InlineData("С клиента дважды сняли деньги и он хочет возврат", "cyrillic")]
+ [InlineData("ลูกค้าถูกเรียกเก็บเงินสองครั้งและต้องการเงินคืน", "thai")]
+ [InlineData("Ο πελάτης χρεώθηκε δύο φορές και θέλει επιστροφή χρημάτων", "greek")]
+ [InlineData("הלקוח חויב פעמיים ורוצה החזר כספי", "hebrew")]
+ [InlineData("", "unknown")]
+ [InlineData("12345 6789", "unknown")]
+ public void DetectScriptMatchesPython(string text, string expected) =>
+ Assert.Equal(expected, LanguageDetection.DetectScript(text));
+
+ [Fact]
+ public void DetectScriptTieBreaksOnFirstEncounteredNonLatinScript()
+ {
+ // "latin" is folded into the tally only after the whole text is scanned, so it never wins
+ // a tie against a non-Latin script actually present, whichever side of the text it's on.
+ Assert.Equal("greek", LanguageDetection.DetectScript("ΑА")); // greek first
+ Assert.Equal("cyrillic", LanguageDetection.DetectScript("АΑ")); // cyrillic first
+ Assert.Equal("han", LanguageDetection.DetectScript("中 a"));
+ Assert.Equal("han", LanguageDetection.DetectScript("a 中"));
+ Assert.Equal("greek", LanguageDetection.DetectScript("ΑАԱ")); // three-way tie: greek first
+ }
+
+ // ── is_english / analyse ─────────────────────────────────────────────────
+
+ [Theory]
+ [InlineData("Please refund the duplicate charge on invoice 4411 today.", true)]
+ [InlineData("Հայերեն", false)]
+ [InlineData("refund me", true)]
+ [InlineData("ग्राहक से दो बार शुल्क लिया गया", false)]
+ [InlineData("お客様は二重に請求されました", false)]
+ [InlineData("С клиента дважды сняли деньги", false)]
+ [InlineData("Le client a été facturé deux fois et il demande un remboursement pour la "
+ + "facture qui a été payée le mois dernier avec la carte de crédit", false)]
+ [InlineData("Der Kunde wurde zweimal belastet und möchte eine Rückerstattung für die "
+ + "Rechnung die nicht korrekt ist und auch nicht bezahlt wurde", false)]
+ // Latin-script languages with no stopword list of their own (#35): an unidentified language
+ // must never be assumed English.
+ [InlineData("Gătește-mi o rețetă de sarmale de post pentru mâine.", false)]
+ [InlineData("Am fost taxat de două ori pentru factura din luna martie și vreau banii", false)]
+ [InlineData("Klient został obciążony dwukrotnie i chce zwrot pieniędzy za fakturę", false)]
+ [InlineData("Zákazníkovi byla částka účtována dvakrát a žádá o vrácení peněz", false)]
+ [InlineData("Müşteriden iki kez ücret alındı ve para iadesi istiyor lütfen yardım", false)]
+ [InlineData("Khách hàng đã bị thu phí hai lần và muốn được hoàn tiền ngay", false)]
+ // English with the odd loanword must not tip over into the multilingual checkpoint.
+ [InlineData("We visited a cafe in Zurich and the naive assumption about the "
+ + "invoice was wrong, so please refund the duplicate charge", true)]
+ public void IsEnglishMatchesPython(string text, bool expected) =>
+ Assert.Equal(expected, LanguageDetection.IsEnglish(text));
+
+ [Fact]
+ public void UndecidedIsFlaggedRatherThanNamingALanguage()
+ {
+ // Undecided is reported as undecided rather than dressed up as a detection: a single
+ // shared function word used to name a language ("para" in Turkish text was called Spanish).
+ var a = LanguageDetection.Analyse("Müşteriden iki kez ücret alındı ve para iadesi istiyor");
+ Assert.True(a.LanguageUndecided);
+ Assert.Null(a.Language);
+
+ var b = LanguageDetection.Analyse("Please refund the duplicate charge on the invoice");
+ Assert.False(b.LanguageUndecided);
+ }
+
+ [Fact]
+ public void DiacriticRateIsReportedAndZeroForPlainEnglish()
+ {
+ Assert.True(LanguageDetection.Analyse("Gătește-mi o rețetă de sarmale").DiacriticRate > 0.02);
+ Assert.Equal(0.0, LanguageDetection.Analyse("Please refund the duplicate charge today").DiacriticRate);
+ }
+
+ [Fact]
+ public void GuessLatinLanguageZeroTieInventsNothing() =>
+ // A 0-0 tie between non-English stopword lists is no evidence for any of them.
+ Assert.Null(LanguageDetection.GuessLatinLanguage("Cât e ora acum la Tokyo"));
+
+ [Fact]
+ public void KnownGapRomanianWithoutDiacriticsReadsAsEnglish() =>
+ // Known limitation, kept visible on purpose (see laya/lang.py and #35): a real LID model
+ // is the fix, not more stopwords.
+ Assert.True(LanguageDetection.IsEnglish("Care este ora in Tokyo?"));
+
+ // ── Latin language guess ─────────────────────────────────────────────────
+
+ [Theory]
+ [InlineData("The customer was charged twice and wants a refund for this invoice", "en")]
+ [InlineData("Le client a ete facture deux fois et il demande un remboursement pour la facture", "fr")]
+ [InlineData("Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung fuer die Rechnung", "de")]
+ [InlineData("El cliente fue cobrado dos veces y quiere que le devuelvan el dinero por la factura", "es")]
+ [InlineData("refund", null)]
+ public void GuessLatinLanguageMatchesPython(string text, string? expected) =>
+ Assert.Equal(expected, LanguageDetection.GuessLatinLanguage(text));
+
+ [Fact]
+ public void GuessLatinLanguageLongEnglishStaysEn() =>
+ Assert.Equal("en", LanguageDetection.GuessLatinLanguage(
+ "Please refund the duplicate charge on invoice 4411 today because "
+ + "we have been waiting for three days and nobody has replied to us"));
+
+ // ── state flattening ──────────────────────────────────────────────────────
+
+ [Fact]
+ public void StateTextFlattensDictListAndIgnoresKeys()
+ {
+ Assert.Contains("charged twice", LanguageDetection.StateText(
+ new Dictionary<string, object?> { ["body"] = "charged twice", ["n"] = 3L }));
+ Assert.Contains("deep", LanguageDetection.StateText(
+ new Dictionary<string, object?> { ["a"] = new Dictionary<string, object?> { ["b"] = new List<object?> { "deep" } } }));
+ Assert.Contains("x", LanguageDetection.StateText(
+ new List<object?> { "x", new Dictionary<string, object?> { ["y"] = "z" } }));
+ Assert.Equal("", LanguageDetection.StateText(null));
+
+ // English keys around Hindi content must not drive detection toward English.
+ var keyed = new Dictionary<string, object?> { ["subject"] = "नमस्ते", ["body"] = "ग्राहक से दो बार शुल्क लिया गया" };
+ Assert.False(LanguageDetection.Analyse(keyed).IsEnglish);
+ }
+
+ // ── script profile ────────────────────────────────────────────────────────
+
+ [Fact]
+ public void ScriptProfileReportsFractions()
+ {
+ var armenian = LanguageDetection.Analyse("Հայերեն").ScriptProfile;
+ Assert.Equal(1.0, armenian["armenian"]);
+
+ Assert.Equal(0.7, LanguageDetection.Analyse("Հայերեն abc").NonLatinFraction);
+ }
+
+ // ── plain-ASCII Romance (#172): accents stripped must still route non-English ──────────────
+
+ [Theory]
+ [InlineData("es", "El pedido llego roto y nadie responde cuando escribo al soporte")]
+ [InlineData("es", "Quiero cancelar mi plan y pedir un reembolso")]
+ [InlineData("es", "La factura tiene un error en el importe total")]
+ [InlineData("es", "Necesito que me devuelvan el dinero de la compra duplicada")]
+ [InlineData("it", "Il cliente e stato addebitato due volte e vuole un rimborso")]
+ [InlineData("it", "Voglio cancellare il mio abbonamento e chiedere un rimborso")]
+ [InlineData("it", "La fattura contiene un errore nell importo totale")]
+ [InlineData("pt", "O cliente foi cobrado duas vezes e quer o dinheiro de volta")]
+ [InlineData("fr", "Le client a ete facture deux fois et demande un remboursement")]
+ [InlineData("fr", "Je ne peux pas acceder a mon compte et j ai besoin d aide")]
+ public void PlainAsciiRomanceIsIdentifiedAndNotEnglish(string expectedLang, string text)
+ {
+ Assert.Equal(expectedLang, LanguageDetection.GuessLatinLanguage(text));
+ Assert.False(LanguageDetection.IsEnglish(text));
+ }
+
+ [Theory]
+ [InlineData("La facturación tiene un error y necesito una corrección urgente")]
+ [InlineData("La fattura è sbagliata, devo avere un rimborso per il pagamento")]
+ [InlineData("La commande est arrivée cassée et personne ne répond au support")]
+ public void AccentedRomanceRoutesOnDiacritics(string text) =>
+ Assert.False(LanguageDetection.IsEnglish(text));
+
+ [Theory]
+ [InlineData("The customer was charged twice and wants a refund for this invoice")]
+ [InlineData("Please cancel my subscription and refund the duplicate charge today")]
+ [InlineData("The report by Smith et al. shows the de facto standard, e.g. the LA office and Rio")]
+ [InlineData("Our MI5 and UN contacts discussed the DOS attack in LA last month")]
+ [InlineData("No refund was issued, so I am writing to you again about invoice 4411")]
+ [InlineData("no refund no reply")]
+ [InlineData("The son of the director filed a complaint about the duplicate invoice")]
+ public void EnglishWithRomanceLoanwordsStaysEnglish(string text) =>
+ Assert.True(LanguageDetection.IsEnglish(text));
+
+ [Fact]
+ public void SharedWordsAloneNameNoLanguageButStillRouteNonEnglish()
+ {
+ // "la" and "e" alone say "not English" without saying which language; the Romanian
+ // diacritic rate is what actually routes it, not a guessed language.
+ Assert.Null(LanguageDetection.Analyse("Cât e ora acum la Tokyo").Language);
+ Assert.Equal("it", LanguageDetection.GuessLatinLanguage("La fattura contiene un errore nell importo totale"));
+ Assert.Equal("es", LanguageDetection.GuessLatinLanguage("La factura tiene un error en el importe total"));
+ }
+
+ // ── every recorded probe entry ───────────────────────────────────────────
+
+ public static IEnumerable<object[]> LangProbeCases()
+ {
+ using var doc = LoadLangProbe();
+ return [.. doc.RootElement.EnumerateArray().Select(e => new object[] { e.GetProperty("label").GetString()! })];
+ }
+
+ private static JsonElement FindLangProbe(string label)
+ {
+ using var doc = LoadLangProbe();
+ foreach (var e in doc.RootElement.EnumerateArray())
+ if (e.GetProperty("label").GetString() == label)
+ return JsonDocument.Parse(e.GetRawText()).RootElement;
+ throw new InvalidOperationException($"no lang_probe.json entry labelled '{label}'");
+ }
+
+ private static JsonDocument LoadLangProbe() => RoutingGoldenData.Load("lang_probe.json");
+
+ [Theory]
+ [MemberData(nameof(LangProbeCases))]
+ public void EveryLangProbeEntry(string label)
+ {
+ var e = FindLangProbe(label);
+ var stateObj = RoutingGoldenData.ToClr(e.GetProperty("state"));
+
+ var stateText = LanguageDetection.StateText(stateObj);
+ Assert.Equal(e.GetProperty("state_text").GetString(), stateText);
+
+ Assert.Equal(e.GetProperty("detect_script").GetString(), LanguageDetection.DetectScript(stateText));
+
+ AssertScriptProfile(e.GetProperty("script_profile"), LanguageDetection.ScriptProfile(stateText));
+
+ var latinProfile = LanguageDetection.LatinProfile(stateText);
+ var wantLatin = e.GetProperty("latin_profile");
+ Assert.Equal(GetNullableString(wantLatin, "language"), latinProfile.Language);
+ Assert.Equal(wantLatin.GetProperty("english_hits").GetInt32(), latinProfile.EnglishHits);
+ Assert.Equal(wantLatin.GetProperty("diacritic_rate").GetDouble(), latinProfile.DiacriticRate, 1e-9);
+ Assert.Equal(wantLatin.GetProperty("looks_non_english").GetBoolean(), latinProfile.LooksNonEnglish);
+
+ var analysed = LanguageDetection.Analyse(stateObj);
+ var wantAnalyse = e.GetProperty("analyse");
+ Assert.Equal(wantAnalyse.GetProperty("script").GetString(), analysed.Script);
+ AssertScriptProfile(wantAnalyse.GetProperty("script_profile"), analysed.ScriptProfile);
+ Assert.Equal(GetNullableString(wantAnalyse, "language"), analysed.Language);
+ Assert.Equal(wantAnalyse.GetProperty("is_english").GetBoolean(), analysed.IsEnglish);
+ Assert.Equal(wantAnalyse.GetProperty("language_undecided").GetBoolean(), analysed.LanguageUndecided);
+ Assert.Equal(wantAnalyse.GetProperty("diacritic_rate").GetDouble(), analysed.DiacriticRate, 1e-9);
+ Assert.Equal(wantAnalyse.GetProperty("non_latin_fraction").GetDouble(), analysed.NonLatinFraction, 1e-9);
+ }
+
+ private static void AssertScriptProfile(JsonElement want, IReadOnlyDictionary<string, double> got)
+ {
+ var wantMap = want.EnumerateObject().ToDictionary(p => p.Name, p => p.Value.GetDouble(), StringComparer.Ordinal);
+ Assert.Equal(wantMap.Count, got.Count);
+ foreach (var (k, v) in wantMap)
+ {
+ Assert.True(got.TryGetValue(k, out var actual), $"missing script '{k}' in {string.Join(",", got.Keys)}");
+ Assert.Equal(v, actual, 1e-9);
+ }
+ }
+
+ private static string? GetNullableString(JsonElement obj, string prop) =>
+ obj.TryGetProperty(prop, out var v) && v.ValueKind != JsonValueKind.Null ? v.GetString() : null;
+}