Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
namespace Laya;
/// <summary>
/// Dependency-free script and Latin-language detection used to route between Laya checkpoints.
/// Ports <c>laya/lang.py</c>.
/// </summary>
/// <remarks>
/// <para>
/// Routing only needs one decision: is this English Latin text, or is it something the English
/// checkpoint cannot read? Benchmarks on MASSIVE (14 languages) showed the English checkpoint
/// collapsing to near-random on non-Latin scripts, while holding up far better on Latin-script
/// languages. So the signal that matters most is script, and the secondary signal is whether
/// Latin text is English.
/// </para>
/// <para>
/// Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic
/// and is explicitly best-effort: pass an explicit model or language when it is already known.
/// </para>
/// <para>
/// Every method here iterates Unicode <em>code points</em> (via <see cref="Rune"/>), never UTF-16
/// code units, and applies Unicode-category checks rather than ASCII-only ones — matching
/// Python's <c>str.isalpha()</c>, <c>len()</c>, slicing and <c>str.lower()</c> semantics exactly.
/// A UTF-16-unit-based port would split astral characters (emoji, Deseret, Mathematical
/// Alphanumeric Symbols) into two units and disagree with Python on script counts and truncation
/// boundaries.
/// </para>
/// </remarks>
public static class LanguageDetection
{
// Unicode blocks the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
// Order matters: it decides which non-Latin script wins a tie in DetectScript (first
// encountered in the source text wins, and Latin never wins a tie against a script that is
// actually present, because "latin" is appended to the tally only after the scan finishes).
private static readonly (string Name, (int Lo, int Hi)[] Ranges)[] ScriptRanges =
[
("greek", [(0x0370, 0x03FF), (0x1F00, 0x1FFF)]),
("cyrillic", [(0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F)]),
("armenian", [(0x0530, 0x058F)]),
("hebrew", [(0x0590, 0x05FF)]),
("arabic", [(0x0600, 0x06FF), (0x0750, 0x077F), (0x08A0, 0x08FF), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF)]),
("devanagari", [(0x0900, 0x097F), (0xA8E0, 0xA8FF)]),
("bengali", [(0x0980, 0x09FF)]),
("gurmukhi", [(0x0A00, 0x0A7F)]),
("gujarati", [(0x0A80, 0x0AFF)]),
("oriya", [(0x0B00, 0x0B7F)]),
("tamil", [(0x0B80, 0x0BFF)]),
("telugu", [(0x0C00, 0x0C7F)]),
("kannada", [(0x0C80, 0x0CFF)]),
("malayalam", [(0x0D00, 0x0D7F)]),
("sinhala", [(0x0D80, 0x0DFF)]),
("thai", [(0x0E00, 0x0E7F)]),
("lao", [(0x0E80, 0x0EFF)]),
("tibetan", [(0x0F00, 0x0FFF)]),
("myanmar", [(0x1000, 0x109F)]),
("georgian", [(0x10A0, 0x10FF)]),
("ethiopic", [(0x1200, 0x137F)]),
("khmer", [(0x1780, 0x17FF)]),
("hangul", [(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)]),
("kana", [(0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF)]),
("han", [(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF)]),
];
// Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is
// weighted and a margin is required before calling something non-English. Order is load
// bearing: LatinProfile's tie-break walks this list (excluding "en") and the first language
// with the highest score wins.
private static readonly string[] StopOrder = ["en", "fr", "de", "es", "pt", "it", "nl", "sv", "ro", "bn", "az"];
private static readonly Dictionary<string, HashSet<string>> Stop = new(StringComparer.Ordinal)
{
["en"] = new(StringComparer.Ordinal)
{
"the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
"this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
"will", "can", "would", "there", "their", "what", "which", "please", "we", "i", "how",
"why", "when", "where", "who", "whom", "does", "done", "doing", "been", "being",
"could", "should", "might", "must", "shall", "doesn", "didn", "isn", "aren", "wasn",
"weren", "haven", "hasn", "couldn", "wouldn", "won", "dont", "wont", "didnt", "doesnt",
"isnt", "wasnt", "havent", "ive", "youre", "thats", "whats", "any", "some", "many",
"much", "every", "each", "these", "those", "both", "few", "they", "them", "she", "him",
"his", "our", "your", "yours", "into", "onto", "between", "after", "before", "through",
"during", "without", "within", "down", "if", "nor", "than", "then", "because", "while",
"until", "here", "only", "very", "too", "ever", "never", "always", "already", "yet",
},
["fr"] = new(StringComparer.Ordinal)
{
"le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
"pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce",
"et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",
"ma", "ta", "sa", "mes", "tes", "ses", "ces", "deux", "trois", "très", "bien",
"tout", "tous", "toute", "fait", "veux", "veut", "peux", "peut", "dois", "doit",
"merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi",
"alors", "donc",
},
["de"] = new(StringComparer.Ordinal)
{
"der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
"auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",
"aus", "bei", "bitte", "dich", "diese", "diesen", "dieser", "dieses", "dir", "einem",
"einen", "einer", "gibt", "habe", "heute", "ich", "im", "in", "jetzt", "kann", "kannst",
"mein", "meine", "meinem", "meinen", "meiner", "mich", "mir", "nach", "noch", "uns",
"wann", "was", "welche", "wie", "wir", "wird", "wo", "zum", "zur",
},
["es"] = new(StringComparer.Ordinal)
{
"el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
"pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",
"la", "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos",
"ni", "dos", "tres", "fue", "fueron", "ser", "tiene", "tienen", "tengo", "puede",
"pueden", "quiero", "necesito", "hemos", "han", "sobre", "entre", "cuando", "donde",
"porque", "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada", "algo",
"aquí", "hoy", "gracias",
},
["pt"] = new(StringComparer.Ordinal)
{
"os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
"dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela",
"o", "e", "na", "nas", "nos", "ao", "aos", "por", "foi", "era", "ser", "sou",
"tem", "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu",
"sua", "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais",
"já", "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
"olá",
"alguem", "alguém", "antes", "até", "boa", "cadê", "consigo", "depois", "deu", "entao",
"então", "estamos", "estava", "estou", "ficou", "fiz", "gostaria", "ja", "meus", "minhas",
"nao", "nenhum", "nenhuma", "ninguem", "ninguém", "noite", "nossa", "nosso", "obrigada",
"pra", "sao", "tambem", "também", "tarde", "tá", "vc", "vcs", "voce", "voces", "você", "vocês",
},
["it"] = new(StringComparer.Ordinal)
{
"il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
"questo", "questa", "anche", "come", "più", "nella", "alla",
"la", "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi",
"ci", "ne", "ho", "hai", "ha", "abbiamo", "avete", "hanno", "era", "stato", "stata",
"devo", "deve", "devono", "voglio", "vorrei", "mio", "mia", "tuo", "sua", "quando",
"dove", "perche", "molto", "poco", "sempre", "mai", "già", "ancora", "adesso", "oggi",
"ieri", "grazie", "ciao", "scusa",
"nel", "nell", "negli", "sul", "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei",
"delle", "dello", "degli", "agli", "alle", "col",
},
["nl"] = new(StringComparer.Ordinal)
{
"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
"door", "maar", "ook", "worden", "deze", "naar", "wordt",
},
["sv"] = new(StringComparer.Ordinal)
{
"jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska",
"vill", "måste", "utan", "också", "min", "mitt", "mig", "vi", "om", "nästa", "här",
"dessa", "detta", "hittar", "kommer", "säger", "återbetalning", "återbetala", "gör", "göra",
"faktura", "gång", "gånger", "hjälp", "hjälpa", "inställningen", "inställningarna",
"lösenord", "när", "öppnar", "spårningen", "två", "upp", "aterbetalning", "aterbetala",
"behover", "fel", "ganger", "hjalp", "hjalpa", "installningen", "installningarna",
"kraschar", "kvittot", "losenord", "nar", "oppnar", "paket", "skicka", "sparningen",
"tva", "uppdaterats", "blivit", "debiterade", "appen",
},
["ro"] = new(StringComparer.Ordinal)
{
"și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
"ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
"acesta", "aceasta", "mi", "ți", "vă", "nu",
},
["bn"] = new(StringComparer.Ordinal)
{
"ami", "amar", "amake", "amra", "amader", "apni", "apnar", "apnake", "apnara",
"tumi", "tomar", "tomake", "tomra", "tader", "ota", "eita", "oita", "ekta", "ei", "oi",
"ki", "keno", "kivabe", "kibhabe", "kothay", "kokhon", "kobe", "koto", "kintu", "jodi",
"tahole", "ar", "theke", "jonno", "sathe", "shathe", "diye", "niye", "moddhe", "kore",
"korte", "korchi", "korsi", "korbo", "korechi", "koreche", "korun", "koren", "korlam",
"hobe", "hoyeche", "hoise", "hocche", "hoyni", "chai", "chaina", "lagbe", "parchi",
"parbo", "parchina", "peyechi", "paini", "dite", "dilam", "diyechi", "nai", "khub",
"onek", "ekhon", "akhon", "ekhono", "abar", "ekbar", "duibar", "ajke", "kalke", "taka",
"bhalo", "valo", "kharap", "shomossa", "somossa", "dhonnobad", "bhai", "shob", "keu",
"kichu", "bolte", "bolun", "parben", "asbe", "jabe", "pabo", "ferot", "dorkar",
"hoye", "geche", "gese",
},
["az"] = new(StringComparer.Ordinal)
{
"və", "ve", "bir", "bu", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",
"var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",
"hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
"ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz",
},
};
private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
private static readonly HashSet<string> NordicOverlapWords = new(StringComparer.Ordinal)
{ "får", "hej", "ja", "jo", "kommer", "mig", "min", "nej", "om", "skulle", "tack", "vi" };
private static readonly HashSet<string> ShortSwedishWords = new(StringComparer.Ordinal)
{
"åtkomst", "atkomst", "lösenord", "losenord", "fakturan", "betalningen", "inloggningen",
"glömt", "glomt", "behöver", "behover", "återbetalning", "aterbetalning", "kvitto",
"kvittot", "spårningen", "sparningen", "inställningen", "installningen", "felmeddelande", "abonnemanget",
};
private static readonly HashSet<string> EnglishCollisionWords = new(StringComparer.Ordinal)
{ "care", "come", "do", "im", "per", "plus", "son", "todo" };
// Letters that ordinary English does not use. This is the signal that catches a Latin-script
// language for which no stopword list is held at all (Romanian, Polish, Czech, Turkish,
// Baltic, ...).
private const string NonEnDiacriticChars =
"àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" // Western European
+ "ăâîșțşţ" // Romanian
+ "ąćęłńśźż" // Polish
+ "čďěňřšťůž" // Czech / Slovak
+ "őű" // Hungarian
+ "ğı" // Turkish (text is lowercased before matching)
+ "āēģīķļņūž" // Baltic
+ "đ" // Serbo-Croatian / Vietnamese
+ "ə" // Azerbaijani
// Vietnamese: no stopword list of its own, so its letters are the only signal that keeps
// it off the English checkpoint. The set used to stop at "đ" (the comment above already
// claimed Vietnamese coverage), but the tone-marked letters are precomposed codepoints of
// their own, so everyday sentences such as "Cảm ơn bạn nhiều nhé" carried a diacritic rate
// of 0.0 and were routed to the English checkpoint. Mirrors laya/lang.py; text is
// lowercased before matching, so lowercase forms are enough.
+ "ạảấầẩẫậắằẳẵặ" // a-family tone marks
+ "ẹẻẽếềểễệỉị" // e-/i-family tone marks
+ "ọỏốồổỗộớờởỡợụủứừửữự" // o-/u-family tone marks
+ "ỳỵỷỹơư"; // y-family tone marks, and the two letters
// no other listed language uses
private static readonly HashSet<int> NonEnDiacritics =
new(NonEnDiacriticChars.EnumerateRunes().Select(r => r.Value));
// Words that more than one list claims. Matching one says "not English" without saying
// *which* language, so it alone may not name a winner (see LatinProfile).
private static readonly HashSet<string> SharedWords = ComputeSharedWords();
/// <summary>
/// A diacritic rate above this is taken as evidence the text is not English, even when no
/// stopword list matches it.
/// </summary>
private const double NonEnDiacriticRate = 0.02;
private static HashSet<string> ComputeSharedWords()
{
var counts = new Dictionary<string, int>(StringComparer.Ordinal);
foreach (var set in Stop.Values)
foreach (var w in set)
counts[w] = counts.GetValueOrDefault(w) + 1;
var shared = new HashSet<string>(counts.Where(kv => kv.Value > 1).Select(kv => kv.Key), StringComparer.Ordinal);
shared.UnionWith(NordicOverlapWords);
return shared;
}
/// <summary>
/// Flatten a state into the text used for detection: string leaves of a string, dict-like or
/// list-like value, joined with a single space and truncated to <paramref name="maxChars"/>
/// Unicode code points. Keys are ignored, since they are usually English.
/// </summary>
/// <remarks>Recursion stops past 6 levels of nesting, matching Python's <c>_iter_text</c>.</remarks>
public static string StateText(object? state, int maxChars = 4000)
{
var leaves = new List<string>();
CollectText(state, leaves, 0);
return SliceCodePoints(string.Join(" ", leaves), maxChars);
}
private static void CollectText(object? state, List<string> leaves, int depth)
{
if (depth > 6 || state is null) return;
switch (state)
{
case string s:
leaves.Add(s);
return;
// bytes: strict UTF-8, and invalid UTF-8 contributes no text (lang.py:253-255).
case byte[] bytes:
try { leaves.Add(StrictUtf8.GetString(bytes)); }
catch (DecoderFallbackException) { }
return;
// Dict-like before generically enumerable, mirroring PythonJson's dispatch order.
case IEnumerable<KeyValuePair<string, object?>> kvpSeq:
foreach (var kv in kvpSeq) CollectText(kv.Value, leaves, depth + 1);
return;
case System.Collections.IDictionary dict:
foreach (System.Collections.DictionaryEntry entry in dict) CollectText(entry.Value, leaves, depth + 1);
return;
// Any other mapping type (an IReadOnlyDictionary only, say): its values, as Python walks any Mapping.
case System.Collections.IEnumerable mapping when IsMapping(state):
foreach (var entry in mapping)
CollectText(entry!.GetType().GetProperty("Value")!.GetValue(entry), leaves, depth + 1);
return;
// _iter_text walks lists and tuples only. A set, a generator or any other iterable holds no text.
case System.Collections.IList list:
foreach (var item in list) CollectText(item, leaves, depth + 1);
return;
case System.Runtime.CompilerServices.ITuple tuple:
for (var i = 0; i < tuple.Length; i++) CollectText(tuple[i], leaves, depth + 1);
return;
default:
// Numbers, booleans and other scalars contribute nothing, matching _iter_text.
return;
}
}
private static bool IsMapping(object state) =>
state.GetType().GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>));
/// <summary>
/// Dominant script of <paramref name="text"/>: <c>"latin"</c>, <c>"han"</c>,
/// <c>"devanagari"</c>, ... or <c>"unknown"</c> if there are no letters.
/// </summary>
public static string DetectScript(string text)
{
ArgumentNullException.ThrowIfNull(text);
// Non-Latin scripts are tallied in the order first encountered in the text; "latin" is
// appended only after the scan, so it never wins a tie against a script actually present,
// regardless of where in the text the Latin characters fall.
var order = new List<string>();
var counts = new Dictionary<string, int>(StringComparer.Ordinal);
var latin = 0;
foreach (var rune in text.EnumerateRunes())
{
if (!Rune.IsLetter(rune)) continue;
var cp = rune.Value;
if (IsLatin(cp)) { latin++; continue; }
var claimed = false;
foreach (var (name, ranges) in ScriptRanges)
{
if (!InRanges(cp, ranges)) continue;
if (!counts.ContainsKey(name)) order.Add(name);
counts[name] = counts.GetValueOrDefault(name) + 1;
claimed = true;
break;
}
if (!claimed)
{
if (!counts.ContainsKey("other")) order.Add("other");
counts["other"] = counts.GetValueOrDefault("other") + 1;
}
}
counts["latin"] = latin;
order.Add("latin");
var total = counts.Values.Sum();
if (total == 0) return "unknown";
var bestName = order[0];
var bestCount = counts[bestName];
for (var i = 1; i < order.Count; i++)
{
var name = order[i];
if (counts[name] > bestCount) { bestName = name; bestCount = counts[name]; }
}
return bestName;
}
/// <summary>Fraction of alphabetic characters belonging to each detected script.</summary>
public static IReadOnlyDictionary<string, double> ScriptProfile(string text)
{
ArgumentNullException.ThrowIfNull(text);
var counts = new Dictionary<string, int>(StringComparer.Ordinal) { ["latin"] = 0 };
foreach (var rune in text.EnumerateRunes())
{
if (!Rune.IsLetter(rune)) continue;
var cp = rune.Value;
if (IsLatin(cp)) { counts["latin"]++; continue; }
var claimed = false;
foreach (var (name, ranges) in ScriptRanges)
{
if (!InRanges(cp, ranges)) continue;
counts[name] = counts.GetValueOrDefault(name) + 1;
claimed = true;
break;
}
if (!claimed) counts["other"] = counts.GetValueOrDefault("other") + 1;
}
var total = counts.Values.Sum();
if (total == 0) return new Dictionary<string, double>(0);
var result = new Dictionary<string, double>(StringComparer.Ordinal);
foreach (var (k, v) in counts)
if (v != 0) result[k] = (double)v / total;
return result;
}
private static bool InRanges(int cp, (int Lo, int Hi)[] ranges)
{
foreach (var (lo, hi) in ranges)
if (cp >= lo && cp <= hi) return true;
return false;
}
private static bool IsLatin(int cp) =>
cp < 0x02B0 || (cp >= 0x1E00 && cp <= 0x1EFF)
|| (cp >= 0xFF21 && cp <= 0xFF3A) || (cp >= 0xFF41 && cp <= 0xFF5A);
/// <summary>
/// Evidence behind the Latin-script language guess: <see cref="Language"/> (may be
/// <see langword="null"/> when undecided), <see cref="EnglishHits"/>,
/// <see cref="DiacriticRate"/> and <see cref="LooksNonEnglish"/>. <see cref="LanguageDetection.Analyse"/>
/// needs the evidence and not just the verdict, because "undecided" and "English" are
/// different answers and only one of them is safe to send to the English checkpoint.
/// </summary>
/// <param name="Language">
/// The best-effort language code, or <see langword="null"/> when undecided. Only named when
/// it matched at least one word that no other list claims: shared function words alone
/// (<c>la</c>, <c>e</c>, <c>o</c>) identify no particular language.
/// </param>
/// <param name="EnglishHits">How many words matched the English stopword list.</param>
/// <param name="DiacriticRate">
/// Fraction of (lowercased) code points that are letters ordinary English does not use.
/// Not rounded; <see cref="LanguageAnalysis.DiacriticRate"/> rounds this to 4 decimals.
/// </param>
/// <param name="LooksNonEnglish"><see cref="DiacriticRate"/> at or above the detection threshold.</param>
public readonly record struct LatinProfileResult(
string? Language, int EnglishHits, double DiacriticRate, bool LooksNonEnglish);
/// <summary>
/// Evidence behind the Latin-script language guess. See <see cref="LatinProfileResult"/>.
/// </summary>
public static LatinProfileResult LatinProfile(string text)
{
ArgumentNullException.ThrowIfNull(text);
var words = ExtractWords(PythonLower(ReplaceIdentifiers(text).Replace("İ", "i"))).ToList();
var lowered = PythonLower(text);
var diac = 0;
var loweredLen = 0;
foreach (var rune in lowered.EnumerateRunes())
{
loweredLen++;
if (NonEnDiacritics.Contains(rune.Value)) diac++;
}
var diacRate = (double)diac / Math.Max(1, loweredLen);
var nonEnglish = diacRate >= NonEnDiacriticRate;
var wordSet = new HashSet<string>(words, StringComparer.Ordinal);
var nordicOverlap = wordSet.Overlaps(NordicOverlapWords)
&& !wordSet.Any(w => Stop["en"].Contains(w) && !SharedWords.Contains(w));
if (words.Count is > 1 and < 4 && wordSet.Overlaps(ShortSwedishWords))
return new LatinProfileResult("sv", 0, diacRate, nonEnglish);
if (words.Count < 4)
return new LatinProfileResult(null, 0, diacRate, nonEnglish || nordicOverlap);
var scores = new Dictionary<string, int>(StringComparer.Ordinal);
var counts = words.GroupBy(w => w).ToDictionary(group => group.Key, group => group.Count(), StringComparer.Ordinal);
foreach (var lang in StopOrder)
scores[lang] = wordSet.Where(w => Stop[lang].Contains(w))
.Sum(w => EnglishCollisionWords.Contains(w) ? 1 : counts[w]);
var en = scores.GetValueOrDefault("en");
// Only a language that matched at least one word no other list claims may be named.
// Without that condition the top score can be pure overlap. Such a language is dropped
// entirely from the running rather than merely losing the tie, so a lesser score with
// real evidence still gets named, and the text stays undecided when no list has any.
string? bestLg = null;
var best = 0;
foreach (var lang in StopOrder)
{
if (lang == "en") continue;
var evidenced = false;
foreach (var w in wordSet)
if (Stop[lang].Contains(w) && !SharedWords.Contains(w)) { evidenced = true; break; }
if (!evidenced) continue;
var score = scores[lang];
if (bestLg is null || score > best) { bestLg = lang; best = score; }
}
string? language = null;
if (bestLg is not null && best >= Math.Max(2, en + 2))
{
// a non-English language needs a clear margin over English function words
language = bestLg;
}
else if (bestLg == "sv" && wordSet.Contains("inte") && wordSet.Contains("kan")
&& words[0] is "kan" or "jag" or "vi" && en <= 1)
{
language = bestLg;
}
else if (bestLg is not null && nonEnglish && best >= Math.Max(2, en))
{
// Needs two hits here too: one shared function word on the strength of the
// diacritics alone is a guess dressed as a detection.
language = bestLg;
}
else if (en > 0 && (!nonEnglish || (diacRate < 0.06
&& wordSet.Count(w => Stop["en"].Contains(w) && !SharedWords.Contains(w)) >= 2
&& wordSet.Count(w => w.EnumerateRunes().Any(r => NonEnDiacritics.Contains(r.Value))) <= 1)))
{
language = "en";
}
return new LatinProfileResult(language, en, diacRate, nonEnglish || (language is null && nordicOverlap));
}
/// <summary>
/// Best-effort language code for Latin-script text, or <see langword="null"/> when
/// undecided.
/// </summary>
/// <remarks>
/// Scores function-word hits per language and requires the winner to beat English by a
/// margin and to have matched at least one word of its own, so ordinary English is never
/// misrouted and a word of several languages at once names none of them. Short inputs
/// usually return <see langword="null"/> on purpose.
/// </remarks>
public static string? GuessLatinLanguage(string text) => LatinProfile(text).Language;
/// <summary>Full detection result for a state.</summary>
public static LanguageAnalysis Analyse(object? state)
{
var result = AnalyseText(StateText(state));
var leaves = new List<string>();
CollectText(state, leaves, 0);
if (result.Script == "latin" && result.IsEnglish
&& (leaves.Count > 1 || leaves.Any(leaf => leaf.Contains('\n'))))
{
var seen = 0;
foreach (var segment in leaves.SelectMany(leaf => leaf.Split('\n')))
{
if (seen >= 4000) break;
var sample = SliceCodePoints(segment, 4000 - seen);
seen += CodePointCount(sample);
var language = NamedProseLanguage(sample);
if (language is not null)
{
result = result with
{
Language = language, IsEnglish = false, LanguageUndecided = false,
MixedSegment = PythonStr.Strip(sample),
};
break;
}
}
}
if (state is null or string or byte[] || !result.IsEnglish) return result;
LanguageAnalysis? best = null;
var bestCount = -1;
foreach (var leaf in leaves)
{
var candidate = LeafNonEnglish(leaf);
if (candidate is null) continue;
var count = SliceCodePoints(leaf, 4000).EnumerateRunes().Count(Rune.IsLetter);
if (count > bestCount) { bestCount = count; best = candidate; }
}
return best is null ? result : result with
{
Language = best.Language, IsEnglish = false, LanguageUndecided = best.LanguageUndecided,
};
}
// _leaf_non_english: a string value that is itself not safe for the English checkpoint.
private static LanguageAnalysis? LeafNonEnglish(string leaf)
{
var bestN = -1;
LanguageAnalysis? best = null;
foreach (var line in leaf.Split('\n'))
{
// A line this short cannot be selected (see lang.py's note), so skip it first.
if (CodePointCount(line) < 7) continue;
var sample = SliceCodePoints(line, 4000);
if (PythonStr.Strip(sample).Length == 0 || HasCodeSyntax(sample)) continue;
var det = AnalyseText(sample);
if (det.IsEnglish) continue;
// A named language still has to survive NamedProseLanguage, but a veto is not a verdict
// of English: the line falls through to the diacritic branch below.
var named = det.Language is not (null or "en") && NamedProseLanguage(sample) is not null;
var alphaCount = sample.EnumerateRunes().Count(Rune.IsLetter);
if (!named)
{
if (det.Script is not ("latin" or "unknown"))
{
if (!(HasNonLatinWords(sample) && alphaCount >= NonLatinMinLetters)) continue;
}
else if (!(det.DiacriticRate >= NonEnDiacriticRate && ExtractWords(sample).Count >= 4)) continue;
}
if (alphaCount > bestN) { bestN = alphaCount; best = det; }
}
return best;
}
// _shouted: a *cased* segment written entirely in capitals.
private static bool Shouted(string text)
{
var upper = false;
foreach (var rune in text.EnumerateRunes())
{
if (PythonStr.IsLower(rune)) return false;
if (PythonStr.IsUpper(rune)) upper = true;
}
return upper;
}
// _shouted_evidence: whether an all-caps line's evidence for `lang` is more than acronym-shaped tokens.
private static bool ShoutedEvidence(List<string> tokens, string lang, double diacriticRate)
{
if (!tokens.Any(t => CodePointCount(t) >= ShoutedMinWord)) return false;
if (diacriticRate >= NonEnDiacriticRate) return true;
var stop = Stop.GetValueOrDefault(lang);
return stop is not null
&& tokens.Select(PythonLower).Any(w => stop.Contains(w) && CodePointCount(w) >= ShoutedMinStopword);
}
private static string? NamedProseLanguage(string segment)
{
if (PythonStr.Strip(segment).Length == 0 || HasCodeSyntax(segment)) return null;
var prose = string.Join(' ', PythonStr.Split(segment).Where(token => !IsJoinedToken(token)));
var shouted = Shouted(prose);
if (!shouted) prose = BlankUpperRuns(prose);
var tokens = ExtractWords(prose);
if (tokens.Count < 4) return null;
var profile = LatinProfile(prose);
var language = profile.Language;
if (language is null or "en") return null;
if (shouted && !ShoutedEvidence(tokens, language, profile.DiacriticRate)) return null;
return tokens.Select(PythonLower).Distinct().Count(w => Stop[language].Contains(w)) >= 2
? language : null;
}
private static bool NonLatinCarries(double nonLatin, int nonLatinLetters) =>
nonLatin >= NonLatinFraction
|| (nonLatin >= NonLatinMinFraction && nonLatinLetters >= NonLatinMinLetters);
private static LanguageAnalysis AnalyseText(string text)
{
var prof = ScriptProfile(text);
var script = DetectScript(text);
var nonLatin = prof.Count > 0 ? Calibration.Round4(1.0 - prof.GetValueOrDefault("latin", 0.0)) : 0.0;
var nonLatinLetters = (int)Math.Round(nonLatin * text.EnumerateRunes().Count(Rune.IsLetter));
if (script == "latin" && HasNonLatinWords(text) && NonLatinCarries(nonLatin, nonLatinLetters))
script = prof.Where(p => p.Key != "latin").MaxBy(p => p.Value).Key;
if (script == "unknown")
return new LanguageAnalysis("unknown", prof, null, true, true, 0.0, 0.0);
if (script != "latin")
return new LanguageAnalysis(script, prof, null, false, true, 0.0, nonLatin);
var profLat = LatinProfile(text);
var lang = profLat.Language;
// The acronym bar belongs here too, not only in the segment scan (lang.py:689-736). Neither
// bar applies when the state is carried by a caseless script.
if (lang is not (null or "en") && !NonLatinCarries(nonLatin, nonLatinLetters))
{
if (Shouted(text))
{
if (!ShoutedEvidence(ExtractWords(text), lang, profLat.DiacriticRate)) lang = null;
}
else
{
// Mixed-case text: re-take the verdict without the acronym-shaped all-caps runs.
var caps = LetterRuns(text).Select(r => text.Substring(r.Start, r.Length))
.Where(PythonStr.StrIsUpper).ToList();
if (caps.Count > 0 && !caps.Any(c => CodePointCount(c) >= ShoutedMinWord))
{
var blanked = BlankUpperRuns(text);
if (blanked != text) lang = LatinProfile(blanked).Language;
}
}
}
// Undecided is not English. Treating it as English sends every Latin-script language no
// stopwords are held for to the checkpoint that cannot read it, silently.
var undecided = lang is null;
var english = lang == "en" || (undecided && !profLat.LooksNonEnglish);
return new LanguageAnalysis("latin", prof, lang, english, undecided,
Calibration.Round4(profLat.DiacriticRate), nonLatin);
}
/// <summary>Whether the English checkpoint can be expected to read this state.</summary>
public static bool IsEnglish(object? state) => Analyse(state).IsEnglish;
private static bool HasNonLatinWords(string text)
{
string? script = null;
var count = 0;
var upper = false;
foreach (var rune in text.EnumerateRunes())
{
if (PythonStr.IsCombining(rune.Value)) continue;
var cp = rune.Value;
var next = cp < 0x0250 || (cp >= 0x1E00 && cp <= 0x1EFF)
|| (cp >= 0xFF21 && cp <= 0xFF3A) || (cp >= 0xFF41 && cp <= 0xFF5A)
? null : ScriptRanges.FirstOrDefault(s => InRanges(cp, s.Ranges)).Name;
if (next is not null && next == script) { count++; continue; }
if (count >= 2 && !upper) return true;
script = next;
count = next is null ? 0 : 1;
upper = PythonStr.IsUpper(rune);
}
return count >= 2 && !upper;
}
// ── Python-compatible scanners ───────────────────────────────────────────
// `re`'s \w, \s and \d differ from .NET's (marks, connector punctuation, U+001C..U+001F) and
// .NET regexes work on UTF-16 units, not code points, so the patterns lang.py runs are written
// over code-point predicates instead.
private const int ShoutedMinStopword = 4;
private const int ShoutedMinWord = 5;
private const double NonLatinFraction = 0.2;
private const double NonLatinMinFraction = 0.1;
private const int NonLatinMinLetters = 10;
private static int CodePointCount(string s)
{
var count = 0;
foreach (var _ in s.EnumerateRunes()) count++;
return count;
}
/// <summary><c>\w</c>: <c>str.isalnum()</c> or the underscore.</summary>
private static bool IsWordChar(Rune r) => r.Value == '_' || Rune.IsLetter(r) || Rune.IsNumber(r);
/// <summary><c>[^\W_]</c>: <c>str.isalnum()</c>.</summary>
private static bool IsAlnum(Rune r) => Rune.IsLetter(r) || Rune.IsNumber(r);
/// <summary><c>_IDENTIFIER.sub(" ", text)</c>: <c>(?<![\w-])[\w-]*(?:[.@][\w-]+)+</c>.</summary>
private static string ReplaceIdentifiers(string text)
{
var runes = text.EnumerateRunes().ToArray();
static bool Ident(Rune r) => IsWordChar(r) || r.Value == '-';
var sb = new StringBuilder(text.Length);
var i = 0;
while (i < runes.Length)
{
// The lookbehind only lets a match start at the head of a run, which is also what keeps this linear.
if (i == 0 || !Ident(runes[i - 1]))
{
var j = i;
while (j < runes.Length && Ident(runes[j])) j++;
var groups = 0;
while (j < runes.Length && runes[j].Value is '.' or '@')
{
var k = j + 1;
while (k < runes.Length && Ident(runes[k])) k++;
if (k == j + 1) break;
j = k;
groups++;
}
if (groups > 0)
{
sb.Append(' ');
i = j;
continue;
}
}
sb.Append(runes[i]);
i++;
}
return sb.ToString();
}
/// <summary><c>_CODE_LINE.search</c>: <c>[=;{}\[\]]|\w\(</c>.</summary>
private static bool HasCodeSyntax(string line)
{
var prevWord = false;
foreach (var rune in line.EnumerateRunes())
{
if (rune.Value is '=' or ';' or '{' or '}' or '[' or ']' || (rune.Value == '(' && prevWord)) return true;
prevWord = IsWordChar(rune);
}
return false;
}
/// <summary><c>_JOINED.search(token)</c>: <c>[^\W_][._/\\][^\W_]</c>.</summary>
private static bool IsJoinedToken(string token)
{
var runes = token.EnumerateRunes().ToArray();
for (var i = 0; i + 2 < runes.Length; i++)
if (IsAlnum(runes[i]) && runes[i + 1].Value is '.' or '_' or '/' or '\\' && IsAlnum(runes[i + 2]))
return true;
return false;
}
/// <summary><c>_LETTER_RUN.finditer</c>: maximal runs of at least two <c>[^\W\d_]</c> characters (UTF-16 offsets).</summary>
private static List<(int Start, int Length)> LetterRuns(string text)
{
var runs = new List<(int, int)>();
var start = -1;
var count = 0;
var offset = 0;
foreach (var rune in text.EnumerateRunes())
{
if (IsWordRune(rune))
{
if (start < 0) { start = offset; count = 0; }
count++;
}
else
{
if (start >= 0 && count >= 2) runs.Add((start, offset - start));
start = -1;
}
offset += rune.Utf16SequenceLength;
}
if (start >= 0 && count >= 2) runs.Add((start, offset - start));
return runs;
}
/// <summary><c>_LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), text)</c>.</summary>
private static string BlankUpperRuns(string text)
{
var sb = new StringBuilder(text.Length);
var last = 0;
foreach (var (start, length) in LetterRuns(text))
{
if (!PythonStr.StrIsUpper(text.Substring(start, length))) continue;
sb.Append(text, last, start - last).Append(' ');
last = start + length;
}
sb.Append(text, last, text.Length - last);
return sb.ToString();
}
// ── word extraction ──────────────────────────────────────────────────────
/// <summary>
/// Runs of code points matching Python's <c>[^\W\d_]+</c>: word characters (letters, plus
/// non-decimal numeric characters such as Roman numerals) excluding decimal digits and the
/// underscore.
/// </summary>
private static List<string> ExtractWords(string text)
{
var words = new List<string>();
var current = new StringBuilder();
foreach (var rune in text.EnumerateRunes())
{
if (IsWordRune(rune))
{
current.Append(rune);
}
else if (current.Length > 0)
{
words.Add(current.ToString());
current.Clear();
}
}
if (current.Length > 0) words.Add(current.ToString());
return words;
}
private static bool IsWordRune(Rune r)
{
if (Rune.IsLetter(r)) return true;
var category = Rune.GetUnicodeCategory(r);
// Python's \w = str.isalnum() (letters, plus Nd/Nl/No numeric categories) or "_"; \d
// matches only decimal digits (Nd). [^\W\d_]+ therefore keeps letters and the
// non-decimal numeric categories (Nl, No — e.g. Roman numerals, vulgar fractions) but
// excludes Nd and the literal underscore.
return category is UnicodeCategory.LetterNumber or UnicodeCategory.OtherNumber;
}
// ── code-point-aware slicing ─────────────────────────────────────────────
/// <summary>Python's <c>str.lower()</c>; see <see cref="PythonStr.Lower"/>.</summary>
internal static string PythonLower(string text) => PythonStr.Lower(text);
/// <summary>Python's <c>s[:maxCodePoints]</c>: a prefix measured in Unicode code points.</summary>
/// <remarks>Internal rather than private so <see cref="LayaEmail"/> can reuse it for <c>max_chars</c>.</remarks>
internal static string SliceCodePoints(string text, int maxCodePoints)
{
if (maxCodePoints <= 0) return string.Empty;
var sb = new StringBuilder();
var count = 0;
foreach (var rune in text.EnumerateRunes())
{
if (count >= maxCodePoints) break;
sb.Append(rune);
count++;
}
return sb.ToString();
}
}
/// <summary>
/// Full language/script detection result for a state. Every field is populated on every branch
/// (script <c>"unknown"</c>, non-Latin, or Latin), so a caller never needs to guard on which
/// branch produced it.
/// </summary>
/// <param name="Script">
/// <c>"unknown"</c> (no letters), <c>"latin"</c>, or a non-Latin script name such as
/// <c>"devanagari"</c> or <c>"han"</c>.
/// </param>
/// <param name="ScriptProfile">Fraction of alphabetic characters belonging to each detected script.</param>
/// <param name="Language">Best-effort ISO-ish language code for Latin-script text, or <see langword="null"/>.</param>
/// <param name="IsEnglish">Whether the English checkpoint can be expected to read this state.</param>
/// <param name="LanguageUndecided">
/// True when the script is not Latin (language is not applicable), or the script is Latin but no
/// language could be identified.
/// </param>
/// <param name="DiacriticRate">
/// Fraction of lowercased code points that are letters ordinary English does not use. Rounded to
/// 4 decimals. Always 0 outside the Latin-script branch.
/// </param>
/// <param name="NonLatinFraction">
/// <c>1 - (fraction of alphabetic characters that are Latin)</c>, rounded to 4 decimals; 0 when
/// there are no letters at all.
/// </param>
public sealed record LanguageAnalysis(
string Script,
IReadOnlyDictionary<string, double> ScriptProfile,
string? Language,
bool IsEnglish,
bool LanguageUndecided,
double DiacriticRate,
double NonLatinFraction)
{
/// <summary>The foreign line or field that overrides a mostly-English state, otherwise null.</summary>
public string? MixedSegment { get; init; }
}