Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

35.7 KB
Raw Blame
type ScriptRanges = Array<[string, Array<[number, number]>]>;

const SCRIPT_RANGES: ScriptRanges = [
  ["greek", [[0x0370, 0x03ff], [0x1f00, 0x1fff]]],
  ["cyrillic", [[0x0400, 0x052f], [0x2de0, 0x2dff], [0xa640, 0xa69f]]],
  ["armenian", [[0x0530, 0x058f]]],
  ["hebrew", [[0x0590, 0x05ff]]],
  ["arabic", [[0x0600, 0x06ff], [0x0750, 0x077f], [0x08a0, 0x08ff], [0xfb50, 0xfdff], [0xfe70, 0xfeff]]],
  ["devanagari", [[0x0900, 0x097f], [0xa8e0, 0xa8ff]]],
  ["bengali", [[0x0980, 0x09ff]]],
  ["gurmukhi", [[0x0a00, 0x0a7f]]],
  ["gujarati", [[0x0a80, 0x0aff]]],
  ["oriya", [[0x0b00, 0x0b7f]]],
  ["tamil", [[0x0b80, 0x0bff]]],
  ["telugu", [[0x0c00, 0x0c7f]]],
  ["kannada", [[0x0c80, 0x0cff]]],
  ["malayalam", [[0x0d00, 0x0d7f]]],
  ["sinhala", [[0x0d80, 0x0dff]]],
  ["thai", [[0x0e00, 0x0e7f]]],
  ["lao", [[0x0e80, 0x0eff]]],
  ["tibetan", [[0x0f00, 0x0fff]]],
  ["myanmar", [[0x1000, 0x109f]]],
  ["georgian", [[0x10a0, 0x10ff]]],
  ["ethiopic", [[0x1200, 0x137f]]],
  ["khmer", [[0x1780, 0x17ff]]],
  ["hangul", [[0x1100, 0x11ff], [0x3130, 0x318f], [0xac00, 0xd7af]]],
  ["kana", [[0x3040, 0x309f], [0x30a0, 0x30ff], [0x31f0, 0x31ff]]],
  ["han", [[0x3400, 0x4dbf], [0x4e00, 0x9fff], [0xf900, 0xfaff]]],
];

const STOP: Record<string, Set<string>> = {
  en: new Set(["the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
    "this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
    "will", "can", "would", "there", "their", "what", "which", "please", "we", "i",
    // Question words, auxiliaries, pronouns and connectives, so a plain request such as "Why hasn't
    // my card been delivered?" is not left undecided. See _STOP["en"] in laya/lang.py for how they
    // were chosen and which words stay out.
    "how", "why", "when", "where", "who", "whom", "does", "done", "doing", "been", "being",
    "could", "should", "might", "must", "shall", "doesn", "didn", "isn", "aren", "wasn",
    "weren", "haven", "hasn", "couldn", "wouldn", "won", "dont", "wont", "didnt", "doesnt",
    "isnt", "wasnt", "havent", "ive", "youre", "thats", "whats", "any", "some", "many",
    "much", "every", "each", "these", "those", "both", "few", "they", "them", "she", "him",
    "his", "our", "your", "yours", "into", "onto", "between", "after", "before", "through",
    "during", "without", "within", "down", "if", "nor", "than", "then", "because", "while",
    "until", "here", "only", "very", "too", "ever", "never", "always", "already", "yet"]),
  fr: new Set(["le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
    "pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce",
    "et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",
    "ma", "ta", "sa", "mes", "tes", "ses", "ces", "deux", "trois", "très", "bien",
    "tout", "tous", "toute", "fait", "veux", "veut", "peux", "peut", "dois", "doit",
    "merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi",
    "alors", "donc"]),
  de: new Set(["der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
    "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",
    "ich", "wir", "mir", "mich", "dir", "dich", "uns", "mein", "meine", "meinen",
    "meinem", "meiner", "diese", "dieser", "diesen", "dieses", "einen", "einem", "einer",
    "wie", "wo", "wann", "welche", "im", "zum", "zur", "aus", "bei", "nach", "noch", "bitte",
    "heute", "jetzt", "kann", "kannst", "habe", "gibt", "wird",
    // shared with English on purpose: counted for English alone, they outvoted short German
    "in", "was"]),
  es: new Set(["el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
    "pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",
    "la", "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos",
    "ni", "dos", "tres", "fue", "fueron", "ser", "tiene", "tienen", "tengo", "puede",
    "pueden", "quiero", "necesito", "hemos", "han", "sobre", "entre", "cuando", "donde",
    "porque", "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada", "algo",
    "aquí", "hoy", "gracias"]),
  pt: new Set(["os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
    "dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela",
    "o", "e", "na", "nas", "nos", "ao", "aos", "por", "foi", "era", "ser", "sou",
    "tem", "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu",
    "sua", "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais",
    "já", "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
    "olá",
    "você", "vocês", "voce", "voces", "vc", "vcs", "nao", "sao", "ja", "até", "tá", "pra",
    "gostaria", "obrigada", "também", "tambem", "estou", "estamos", "meus", "minhas",
    "nosso", "nossa", "consigo", "cadê", "boa", "tarde", "noite",
    "depois", "antes", "então", "entao", "ninguém", "ninguem", "alguém", "alguem", "nenhum",
    "nenhuma", "estava", "ficou", "fiz", "deu"]),
  it: new Set(["il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
    "questo", "questa", "anche", "come", "più", "sono", "nella", "alla",
    "la", "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi",
    "ci", "ne", "ho", "hai", "ha", "abbiamo", "avete", "hanno", "era", "stato", "stata",
    "devo", "deve", "devono", "voglio", "vorrei", "mio", "mia", "tuo", "sua", "quando",
    "dove", "perche", "molto", "poco", "sempre", "mai", "già", "ancora", "adesso", "oggi",
    "ieri", "grazie", "ciao", "scusa",
    "nel", "nell", "negli", "sul", "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei",
    "delle", "dello", "degli", "agli", "alle", "col"]),
  nl: new Set(["het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
    "door", "maar", "ook", "worden", "deze", "naar", "wordt"]),
  // Keep Swedish routing in parity with Python. Distinctive words help ASCII-normalised ticket text;
  // shared Nordic words are tracked separately so Danish and Norwegian are not named Swedish.
  sv: new Set(["jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska",
    "vill", "måste", "också", "dessa", "detta", "säger", "upp", "utan", "mitt", "min", "om",
    "kommer", "här", "två", "vi", "nästa", "gör", "göra", "hjälp", "hjälpa", "mig",
    "återbetalning", "återbetala", "faktura", "gång", "gånger", "hittar", "inställningen",
    "inställningarna", "lösenord", "när", "öppnar", "spårningen", "aterbetalning", "aterbetala",
    "behover", "fel", "ganger", "hjalp", "hjalpa", "installningen", "installningarna", "kraschar",
    "kvittot", "losenord", "nar", "oppnar", "paket", "skicka", "sparningen", "tva", "uppdaterats",
    "blivit", "debiterade", "appen"]),
  ro: new Set(["și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
    "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
    "acesta", "aceasta", "mi", "ți", "vă", "nu"]),
  // Romanized Bangla ("Banglish"): how Bangla is typed in chats, tickets and email when no Bengali
  // keyboard is at hand. It has no diacritics, so without a list it read as undecided-but-English
  // and went to the English checkpoint, which scores 0.08 on Bangla at 0.94 confidence.
  bn: new Set(["ami", "amar", "amake", "amra", "amader", "apni", "apnar", "apnake", "apnara",
    "tumi", "tomar", "tomake", "tomra", "tader", "ota", "eita", "oita",
    "ekta", "ei", "oi", "ki", "keno", "kivabe", "kibhabe", "kothay", "kokhon", "kobe",
    "koto", "kintu", "jodi", "tahole", "ar", "theke", "jonno", "sathe", "shathe", "diye",
    "niye", "moddhe", "kore", "korte", "korchi", "korsi", "korbo", "korechi", "koreche",
    "korun", "koren", "korlam", "hobe", "hoyeche", "hoise", "hocche", "hoyni",
    "chai", "chaina", "lagbe", "parchi", "parbo", "parchina", "peyechi", "paini",
    "dite", "dilam", "diyechi", "nai", "khub", "onek", "ekhon", "akhon", "ekhono",
    "abar", "ekbar", "duibar", "ajke", "kalke", "taka", "bhalo", "valo", "kharap",
    "shomossa", "somossa", "dhonnobad", "bhai", "shob", "keu", "kichu", "bolte", "bolun",
    "parben", "asbe", "jabe", "pabo", "ferot", "dorkar", "hoye", "geche", "gese"]),
  az: new Set(["və", "ve", "bir", "bu", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",
    "var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",
    "hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
    "ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"]),
};

const NON_EN_DIACRITICS = new Set(
  ("àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" +   // Western European
    "ăâîșțşţ" +                            // Romanian
    "ąćęłńśźż" +                           // Polish
    "čďěňřšťůž" +                          // Czech / Slovak
    "őű" +                                 // Hungarian
    "ğı" +                                 // Turkish (text is lowercased before matching)
    "āēģīķļņūž" +                          // Baltic
    "đ" +                                  // Serbo-Croatian / Vietnamese
    "ə" +                                  // Azerbaijani
    // Vietnamese: no stopword list of its own, so its letters are the only signal that keeps
    // it off the English checkpoint, and the tone-marked letters are precomposed codepoints
    // of their own (see laya/lang.py for the same fix).
    "ạảấầẩẫậắằẳẵặ" +
    "ẹẻẽếềểễệỉị" +
    "ọỏốồổỗộớờởỡợụủứừửữự" +
    "ỳỵỷỹơư").split(""),
);

export const NON_EN_DIACRITIC_RATE = 0.02;
const ENGLISH_RESCUE_DIACRITIC_RATE = 0.06;

// One accented loanword (`café`, `José`) clears the rate above on its own; two English-only function words
// and at most one accented word keep the text English, as in laya/lang.py.
function englishRescuedByWords(wordSet: Set<string>, diacRate: number): boolean {
  if (diacRate >= ENGLISH_RESCUE_DIACRITIC_RATE) return false;
  let englishOnly = 0;
  let accented = 0;
  for (const w of wordSet) {
    if (EN_ONLY_WORDS.has(w)) englishOnly += 1;
    if ([...w].some((ch) => NON_EN_DIACRITICS.has(ch))) accented += 1;
  }
  return englishOnly >= 2 && accented <= 1;
}

const SHARED_WORDS: Set<string> = (() => {
  const counts = new Map<string, number>();
  for (const words of Object.values(STOP)) {
    for (const w of words) counts.set(w, (counts.get(w) ?? 0) + 1);
  }
  const out = new Set<string>();
  for (const [w, n] of counts) if (n > 1) out.add(w);
  return out;
})();
// Danish and Norwegian share many common words with Swedish; they can add score but cannot name sv alone.
const NORDIC_OVERLAP_WORDS = new Set(["hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi"]);
for (const word of NORDIC_OVERLAP_WORDS) SHARED_WORDS.add(word);
const EN_ONLY_WORDS = new Set([...STOP["en"]].filter((w) => !SHARED_WORDS.has(w)));
// Foreign function words that are also ordinary English words (`im` is also `I'm` without the
// apostrophe). Each counts once however often it repeats, so "do more, do less" cannot clear the
// `best >= 2` bar on one word. Every other word keeps counting occurrences: `der` twice is still
// German, and Dutch `van` is deliberately left out. See _EN_COLLISION_WORDS in laya/lang.py.
const EN_COLLISION_WORDS = new Set(
  ["come", "son", "do", "care", "todo", "im", "per", "plus"].filter((w) =>
    Object.entries(STOP).some(([lg, sw]) => lg !== "en" && sw.has(w))),
);

// Short support fragments need a narrower vocabulary than the general four-word language guess.
// Generic words such as "fel" and "hjälp" are deliberately omitted from this subset.
const SHORT_SWEDISH_WORDS = new Set([
  "åtkomst", "atkomst", "lösenord", "losenord", "fakturan", "betalningen", "inloggningen",
  "glömt", "glomt", "behöver", "behover", "återbetalning", "aterbetalning", "kvitto", "kvittot",
  "spårningen", "sparningen", "inställningen", "installningen", "felmeddelande", "abonnemanget",
]);

// JS `\w` is ASCII-only, so Python's `[^\W\d_]` needs the Unicode classes spelled out (Nl/No are in Python's `\w`).
const WORD_RE = /[\p{L}\p{Nl}\p{No}]+/gu;
// Lookbehind for the same reason as the Python side (see laya/lang.py): without it the
// greedy prefix is retried at every offset inside a run of word characters, which is
// quadratic in the run's length -- 50 000 characters of one token took 1540 ms here.
// It removes no match, because a leftmost match can only begin at a run start.
const IDENTIFIER_RE = /(?<![\p{L}\p{N}_-])[\p{L}\p{N}_-]*(?:[.@][\p{L}\p{N}_-]+)+/gu;
const IS_ALPHA_RE = /\p{L}/u;

// Python uses two different boundaries on purpose: detect_script/script_profile count the IPA
// Extensions block (0x0250-0x02AF) as Latin, while _script_of (used for non-Latin word runs)
// stops at 0x0250 so a pronunciation like [vlɐˈdʲimʲɪr] is neither Latin nor a script run.
function isLatinCp(cp: number): boolean {
  return cp < 0x02b0 || (0x1e00 <= cp && cp <= 0x1eff) || (0xff21 <= cp && cp <= 0xff3a) || (0xff41 <= cp && cp <= 0xff5a);
}

function scriptOf(ch: string): string | null {
  const cp = ch.codePointAt(0)!;
  if (cp < 0x0250 || (0x1e00 <= cp && cp <= 0x1eff) || (0xff21 <= cp && cp <= 0xff3a) || (0xff41 <= cp && cp <= 0xff5a)) {
    return null;
  }
  for (const [name, ranges] of SCRIPT_RANGES) {
    if (ranges.some(([lo, hi]) => lo <= cp && cp <= hi)) return name;
  }
  return null;
}

/** Python's `len()`: the number of code points, not of UTF-16 units. Python's `lang.py` divides
 * the diacritic count by `len(lowered)`, and every other port counts code points too (Go
 * `utf8.RuneCountInString`, Rust `chars().count()`, Java `codePointLength`, C++ `RuneCount`), so
 * an astral character must count as one here as well. `String.length` counts it as two, which
 * understated `diacriticRate` on any text carrying emoji -- e.g. `analyse()` on
 * `"é\uD83D\uDE00\uD83D\uDE00" + "-".repeat(46)` read `isEnglish: true` in this port where
 * Python answers `false`, routing the text to the English checkpoint. */
function width(text: string): number {
  let count = 0;
  for (const _ch of text) count++;
  return count;
}

/** Python's `text[:count]`: a prefix cut at code points, so a surrogate pair is never split
 * in half. Python's `lang.py` slices the same budgets by code points (`state_text`,
 * `_non_english_segment`, the 4000-character leaf samples), and the email port already cuts this
 * way (`cutCodePoints` in `email.ts`, added when a budget cut an emoji in half). `String.slice`
 * counts UTF-16 units, so `stateText("A".repeat(3999) + "\uD83D\uDE00" + "...")` used to return a
 * string ending in the orphaned high half of the emoji -- text Python never produces, and a lone
 * surrogate the serve side already refuses as malformed (#454, #814). */
function cutCodePoints(text: string, count: number): string {
  const chars = Array.from(text);
  const end = count < 0 ? Math.max(0, chars.length + count) : Math.min(count, chars.length);
  return chars.slice(0, end).join("");
}

/** String leaves of a state. Keys are ignored: they are usually English field names. */
function iterText(state: unknown, depth = 0): string[] {
  if (depth > 6 || state == null) return [];
  if (typeof state === "string") return [state];
  if (state instanceof Uint8Array) {
    try {
      return [new TextDecoder("utf-8", { fatal: true }).decode(state)];
    } catch {
      return [];
    }
  }
  if (Array.isArray(state)) {
    const out: string[] = [];
    for (const v of state) out.push(...iterText(v, depth + 1));
    return out;
  }
  if (typeof state === "object") {
    const out: string[] = [];
    for (const v of Object.values(state as object)) out.push(...iterText(v, depth + 1));
    return out;
  }
  return [];
}

export function stateText(state: unknown, maxChars = 4000): string {
  return cutCodePoints(iterText(state).join(" "), maxChars);
}

export function detectScript(text: string): string {
  const counts = new Map<string, number>();
  let latin = 0;
  for (const ch of text) {
    if (!IS_ALPHA_RE.test(ch)) continue;
    const cp = ch.codePointAt(0)!;
    if (isLatinCp(cp)) { latin += 1; continue; }
    let found: string | null = null;
    for (const [name, ranges] of SCRIPT_RANGES) {
      if (ranges.some(([lo, hi]) => lo <= cp && cp <= hi)) { found = name; break; }
    }
    counts.set(found ?? "other", (counts.get(found ?? "other") ?? 0) + 1);
  }
  counts.set("latin", latin);
  let total = 0;
  for (const v of counts.values()) total += v;
  if (total === 0) return "unknown";
  let best = "latin";
  let bestN = -1;
  for (const [k, v] of counts) {
    if (v > bestN) { bestN = v; best = k; }
  }
  return best;
}

export function scriptProfile(text: string): Record<string, number> {
  const counts = new Map<string, number>([["latin", 0]]);
  for (const ch of text) {
    if (!IS_ALPHA_RE.test(ch)) continue;
    const cp = ch.codePointAt(0)!;
    if (isLatinCp(cp)) { counts.set("latin", (counts.get("latin") ?? 0) + 1); continue; }
    let found: string | null = null;
    for (const [name, ranges] of SCRIPT_RANGES) {
      if (ranges.some(([lo, hi]) => lo <= cp && cp <= hi)) { found = name; break; }
    }
    const key = found ?? "other";
    counts.set(key, (counts.get(key) ?? 0) + 1);
  }
  let total = 0;
  for (const v of counts.values()) total += v;
  if (!total) return {};
  const out: Record<string, number> = {};
  for (const [k, v] of counts) {
    if (v) out[k] = v / total;
  }
  return out;
}

// Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a
// brand name or order code outvotes the CJK request around it letter for letter, though one CJK
// character carries far more than a letter. A short message needs a large share to count; a long
// payload (ticket fields, English agent turns) dilutes the share, so there a sentence's worth of
// letters counts too.
// All-caps handling. A customer shouting in Portuguese is still Portuguese; an acronym block is not
// French. A token of this many LETTERS is a word rather than an acronym, and on an all-caps line it
// has to be a function word of the named language (or the line has to carry non-English
// diacritics) before the line counts as prose in that language.
const SHOUTED_MIN_WORD = 5;
const SHOUTED_MIN_STOPWORD = 4;

const NON_LATIN_FRACTION = 0.2;
const NON_LATIN_MIN_FRACTION = 0.1;
const NON_LATIN_MIN_LETTERS = 10;

/** Non-Latin runs that read as words rather than as annotation inside English prose. */
function nonLatinWords(text: string): string[] {
  // English prose carries three kinds of non-Latin letters that are not a request written in
  // another script, and each is excluded here: a symbol ("Set α to 0.05", one letter), a proper
  // name (capitalised), and a pronunciation ([vlɐˈdʲimʲɪr], which no script range claims). A
  // combining mark belongs to the letter before it and never splits a word.
  const runs: string[] = [];
  let cur = "";
  let script: string | null = null;
  for (const ch of text) {
    if (/^\p{M}$/u.test(ch)) continue;
    const s = scriptOf(ch);
    if (s !== null && s === script) {
      cur += ch;
      continue;
    }
    if (cur) runs.push(cur);
    if (s !== null) {
      cur = ch;
      script = s;
    } else {
      cur = "";
      script = null;
    }
  }
  if (cur) runs.push(cur);
  return runs.filter((w) => [...w].length >= 2 && !/^[\p{Lu}\p{Lt}]/u.test(w));
}

export interface LatinProfile {
  language: string | null;
  englishHits: number;
  diacriticRate: number;
  looksNonEnglish: boolean;
}

export function latinProfile(text: string): LatinProfile {
  const stripped = text.replace(IDENTIFIER_RE, " ");
  const rawWords = stripped.match(WORD_RE) ?? [];
  const words = rawWords.map((w) => w.toLowerCase());
  const lowered = text.toLowerCase();
  let diac = 0;
  for (const ch of lowered) {
    if (NON_EN_DIACRITICS.has(ch)) diac += 1;
  }
  const diacRate = diac / Math.max(1, width(lowered));
  const nonEnglish = diacRate >= NON_EN_DIACRITIC_RATE;
  const nordicOverlap = words.some((w) => NORDIC_OVERLAP_WORDS.has(w)) &&
    !words.some((w) => EN_ONLY_WORDS.has(w));
  if (words.length > 1 && words.length < 4 && words.some((w) => SHORT_SWEDISH_WORDS.has(w))) {
    return { language: "sv", englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish };
  }
  if (words.length < 4) {
    return { language: null, englishHits: 0, diacriticRate: diacRate, looksNonEnglish: nonEnglish || nordicOverlap };
  }
  // A collision word counts once however often it repeats; every other word counts its hits.
  const counts = new Map<string, number>();
  for (const w of words) counts.set(w, (counts.get(w) ?? 0) + 1);
  const scores: Record<string, number> = {};
  for (const [lg, sw] of Object.entries(STOP)) {
    let s = 0;
    for (const [w, n] of counts) if (sw.has(w)) s += EN_COLLISION_WORDS.has(w) ? 1 : n;
    scores[lg] = s;
  }
  const en = scores["en"] ?? 0;
  const wordSet = new Set(words);
  let bestLg: string | null = null;
  let best = 0;
  for (const [lg, s] of Object.entries(scores)) {
    if (lg === "en") continue;
    const hasEvidence = [...wordSet].some((w) => STOP[lg].has(w) && !SHARED_WORDS.has(w));
    if (!hasEvidence) continue;
    if (s > best) { best = s; bestLg = lg; }
  }
  let lang: string | null = null;
  if (bestLg && best >= Math.max(2, en + 2)) {
    lang = bestLg;
  } else if (bestLg === "sv" && words.includes("inte") && words.includes("kan") &&
      ["kan", "jag", "vi"].includes(words[0]) && en <= 1) {
    // Short login requests like "kan inte logga in" contain one English-shaped token (`in`).
    lang = bestLg;
  } else if (bestLg && nonEnglish && best >= Math.max(2, en)) {
    lang = bestLg;
  } else if (en && (!nonEnglish || englishRescuedByWords(wordSet, diacRate))) {
    lang = "en";
  }
  return { language: lang, englishHits: en, diacriticRate: diacRate, looksNonEnglish: nonEnglish || (lang === null && nordicOverlap) };
}

export function guessLatinLanguage(text: string): string | null {
  return latinProfile(text).language;
}

export interface AnalyseResult {
  script: string;
  scriptProfile: Record<string, number>;
  language: string | null;
  isEnglish: boolean;
  languageUndecided: boolean;
  diacriticRate: number;
  nonLatinFraction: number;
  mixedSegment: string | null;
}

// Code is not prose in any language, but split into words it reads as one: `os.path` is Portuguese
// (`os`), `round(el, 2)` Spanish (`el`), `non_english` Italian (`non`). A line pasted from a program
// into an English request must not count as a foreign segment, so a line carrying code syntax --
// `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers
// are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.
const CODE_LINE_RE = /[=;{}[\]]|\w\(/;
// Slash and backslash compounds are names, not sentences: `Nav/Com` and `OS/2` read as Portuguese
// (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`). A whitespace token
// holding a letter or digit, a joiner (`.`, `_`, `/`, `\`) and another letter or digit is an
// identifier or a compound and is dropped whole.
const JOINED_RE = /[^\W_][._/\\][^\W_]/;
// An all-caps token inside mixed-case text is an acronym or a code: `MON`, `LA`, `EST`, `COM`, `DES`
// are hockey teams, states, time zones and radio bands, not French or Portuguese. A segment written
// entirely in capitals keeps its words -- a customer shouting in Portuguese is still Portuguese.
const LETTER_RUN_RE = /[\p{L}]{2,}/gu;

/** True for a CASED segment written entirely in capitals.
 *
 * The uppercase half is not redundant: a caseless script has no lowercase either, so testing only
 * "has a lowercase letter" would call every Devanagari or CJK line shouted and hold it to a bar it
 * cannot clear. Both caps bars stay off text that is partly caseless (`analyseText`), because a bar
 * that works by discarding Latin evidence has nothing to say about the half it cannot read.
 */
function shouted(text: string): boolean {
  let upper = false;
  for (const ch of text) {
    if (/\p{Ll}/u.test(ch)) return false;
    if (/\p{Lu}|\p{Lt}/u.test(ch)) upper = true;
  }
  return upper;
}

/** Whether an all-caps line's evidence for `lang` is more than acronym-shaped tokens. */
function shoutedEvidence(tokens: string[], lang: string, diacriticRate: number): boolean {
  if (!tokens.some((t) => width(t) >= SHOUTED_MIN_WORD)) return false;
  if (diacriticRate >= NON_EN_DIACRITIC_RATE) return true;
  const stopSet = STOP[lang];
  if (!stopSet) return false;
  const matched = new Set(tokens.map((t) => t.toLowerCase()).filter((t) => stopSet.has(t)));
  return [...matched].some((w) => width(w) >= SHOUTED_MIN_STOPWORD);
}

/** Language code for one non-code line, or null when it does not name a foreign language.
 *
 * Same evidence bar as `nonEnglishSegment`: four words, a language `latinProfile` will name,
 * and two *different* words of that language. Acronyms and slash compounds are not words.
 */
function namedProseLanguage(segment: string): string | null {
  if (!segment.trim() || CODE_LINE_RE.test(segment)) return null;
  let prose = segment.split(/\s+/).filter((tok) => !JOINED_RE.test(tok)).join(" ");
  // A shouted segment keeps its acronym-shaped tokens -- shouting is not a foreign language, so it
  // must also clear `shoutedEvidence` below. Mixed-case text blanks all-caps runs (they are
  // acronyms) before the words are counted, which is Python's `_named_prose_language` split.
  const isShouted = shouted(prose);
  if (!isShouted) {
    prose = prose.replace(LETTER_RUN_RE, (m) => (m === m.toUpperCase() ? " " : m));
  }
  const tokens = prose.match(WORD_RE) ?? [];
  if (tokens.length < 4) return null;
  const prof = latinProfile(prose);
  const lang = prof.language;
  if (lang === null || lang === "en") return null;
  if (isShouted && !shoutedEvidence(tokens, lang, prof.diacriticRate)) return null;
  const stopSet = STOP[lang];
  if (!stopSet) return null;
  const distinctHits = new Set(tokens.map((w) => w.toLowerCase()).filter((w) => stopSet.has(w)));
  if (distinctHits.size < 2) return null;
  return lang;
}

/** First line or field that, read on its own, is named a non-English language, else null.
 *
 * Returns [language, segment]. A segment needs the evidence a whole state needs -- at least four
 * words, and a language named by `latinProfile` -- and, because one line carries far less text
 * than a state, two things more: the words that name the language must be two *different* ones
 * (`COM ... COM` in an English radio listing is one word seen twice), and acronyms and slash
 * compounds are not words. This adds no new way to call English text foreign; it only stops a
 * longer English part from outvoting a foreign one. Reads at most `maxChars` characters in all.
 */
function nonEnglishSegment(state: unknown, maxChars = 4000): [string, string] | null {
  let seen = 0;
  for (const leaf of iterText(state)) {
    for (const seg of leaf.split("\n")) {
      if (seen >= maxChars) return null;
      const capped = cutCodePoints(seg, maxChars - seen);
      seen += [...capped].length;
      const lang = namedProseLanguage(capped);
      if (lang) return [lang, capped.trim()];
    }
  }
  return null;
}

/** Whether the non-Latin share of a Latin-plurality text is a real message, not a stray name. */
function nonLatinCarries(nonLatin: number, nNonLatin: number): boolean {
  return nonLatin >= NON_LATIN_FRACTION ||
    (nonLatin >= NON_LATIN_MIN_FRACTION && nNonLatin >= NON_LATIN_MIN_LETTERS);
}

// Python's round(x, 4) rounds the exact binary value of the double, breaking a tie to the even
// digit. Math.round instead rounds `x * 10000`, which differs in two ways: it breaks a tie away
// from zero, and the multiplication invents a tie where the double is not one. Both are reachable
// from a ratio of small counts. 1 diacritic in 32 characters is 0.03125 exactly, and Python
// reports 0.0312 where Math.round reported 0.0313. 1 in 4000 is 0.00025 only to the decimal
// reader -- the double is 0.0002500000000000000052, above the halfway point, so Python reports
// 0.0003 while `0.00025 * 10000` lands on 2.5 exactly and a tie rule would report 0.0002. So the
// digits are taken from the double's own decimal expansion, where "tie" means what it means to
// Python, and only a real tie consults the parity of the digit before it.
function round4(x: number): number {
  const [whole, fraction = ""] = x.toFixed(20).split(".");
  const kept = Number(whole + fraction.slice(0, 4).padEnd(4, "0"));
  const rest = fraction.slice(4);
  const first = rest.charAt(0);
  const tail = rest.slice(1).replace(/0+$/, "");
  if (first > "5" || (first === "5" && (tail.length > 0 || kept % 2 === 1))) {
    return (kept + 1) / 10000;
  }
  return kept / 10000;
}

function analyseText(text: string): AnalyseResult {
  const prof = scriptProfile(text);
  let script = detectScript(text);
  const nonLatin = prof && Object.keys(prof).length ? round4(1.0 - (prof["latin"] ?? 0.0)) : 0.0;
  let nAlpha = 0;
  for (const ch of text) if (IS_ALPHA_RE.test(ch)) nAlpha += 1;
  const nNonLatin = Math.round(nonLatin * nAlpha);
  if (script === "latin" && nonLatinWords(text).length > 0 && nonLatinCarries(nonLatin, nNonLatin)) {
    // The plurality said Latin, but the non-Latin share is a real message, not a stray name or
    // symbol: re-classify to the dominant non-Latin script, as Python's analyse does.
    let bestScript = script;
    let bestShare = -1;
    for (const [s, share] of Object.entries(prof)) {
      if (s !== "latin" && share > bestShare) {
        bestShare = share;
        bestScript = s;
      }
    }
    script = bestScript;
  }
  if (script === "unknown") {
    return {
      script: "unknown", scriptProfile: prof, language: null,
      isEnglish: true, languageUndecided: true, diacriticRate: 0.0,
      nonLatinFraction: 0.0, mixedSegment: null,
    };
  }
  if (script !== "latin") {
    return {
      script, scriptProfile: prof, language: null,
      isEnglish: false, languageUndecided: true, diacriticRate: 0.0,
      nonLatinFraction: nonLatin, mixedSegment: null,
    };
  }
  const profLat = latinProfile(text);
  let lang = profLat.language;
  // The acronym bar belongs here too, not only in the segment scan. That scan runs once a state
  // already reads English overall, so a state that is *nothing but* an acronym line -- or one
  // diluted by fewer English lines than it takes to tip this verdict -- never reached it and was
  // named foreign outright: `MON DES EST LA` read French on its own. Vetoing here leaves the text
  // undecided, so `looksNonEnglish` still decides it, and a shouted line with non-English
  // diacritics is kept.
  //
  // Neither bar applies when the state is carried by a caseless script: both answer the verdict by
  // taking evidence away from the Latin letters, and what is left is judged on Latin diacritics,
  // which cannot see the caseless half -- so `ПРИВЕТ MON DES EST LA` would otherwise be sent to the
  // English checkpoint on the strength of disbelieving the Latin part.
  if (lang !== null && lang !== "en" && !nonLatinCarries(nonLatin, nNonLatin)) {
    if (shouted(text)) {
      if (!shoutedEvidence(text.match(WORD_RE) ?? [], lang, profLat.diacriticRate)) lang = null;
    } else {
      // Mixed-case text: re-take the verdict without the all-caps runs that voted in it. Only
      // ACRONYM-SHAPED runs are blanked -- a run of SHOUTED_MIN_WORD letters is a word, and a word
      // is not blanked out of its own sentence (blanking every run deleted emphasis capitals, so
      // `sag mir das HEUTIGE DATUM` lost its verdict). Diacritic evidence stays measured on the
      // original text, so a shouted foreign word cannot be blanked out of the safety net.
      const caps = (text.match(LETTER_RUN_RE) ?? []).filter((m) => m === m.toUpperCase());
      if (caps.length > 0 && !caps.some((c) => width(c) >= SHOUTED_MIN_WORD)) {
        const blanked = text.replace(LETTER_RUN_RE, (m) => (m === m.toUpperCase() ? " " : m));
        if (blanked !== text) lang = latinProfile(blanked).language;
      }
    }
  }
  const undecided = lang === null;
  const english = lang === "en" || (undecided && !profLat.looksNonEnglish);
  return {
    script: "latin", scriptProfile: prof, language: lang,
    isEnglish: english, languageUndecided: undecided,
    diacriticRate: round4(profLat.diacriticRate),
    nonLatinFraction: nonLatin, mixedSegment: null,
  };
}

function alphaCount(text: string): number {
  let n = 0;
  for (const ch of text) if (IS_ALPHA_RE.test(ch)) n += 1;
  return n;
}

/** A string value that is itself not safe for the English checkpoint, else null. */
function leafNonEnglish(leaf: string): AnalyseResult | null {
  const sample = cutCodePoints(leaf, 4000);
  if (!sample.trim()) return null;
  const det = analyseText(sample);
  if (det.isEnglish) return null;
  if (det.language !== null && det.language !== "en") return det;
  const nAlpha = alphaCount(sample);
  if (det.script !== "latin" && det.script !== "unknown") {
    if (nonLatinWords(sample).length > 0 && nAlpha >= NON_LATIN_MIN_LETTERS) return det;
    return null;
  }
  const words = sample.match(WORD_RE) ?? [];
  if (det.languageUndecided && det.diacriticRate >= NON_EN_DIACRITIC_RATE && words.length >= 4) {
    return det;
  }
  return null;
}

export function analyse(state: unknown): AnalyseResult {
  // One non-English string value is enough. Joining every value let a long English
  // note fill the window, or outvote a short German message, and that message was
  // then sent to the English checkpoint.
  const result = analyseText(stateText(state));
  if (result.script === "latin" && result.isEnglish) {
    // A Portuguese ticket with an English stack trace, error payload or form template reads as
    // English as a whole, because the English part is longer -- yet the part a question is about
    // is the customer's, and the English checkpoint cannot read it. So a state that would go to
    // English is checked line by line and field by field.
    const leaves = iterText(state);
    // a single line has no other part to be outvoted by, and was just read whole
    if (leaves.length > 1 || leaves.some((leaf) => leaf.includes("\n"))) {
      const found = nonEnglishSegment(state);
      if (found) {
        const [lang, mixed] = found;
        return {
          ...result,
          language: lang,
          isEnglish: false,
          languageUndecided: false,
          mixedSegment: mixed,
        };
      }
    }
  }
  // A plain string was just read whole. A structured state can still hide a message past the
  // segment cap, or in a script `latinProfile` does not name.
  if (typeof state === "string" || state == null || state instanceof Uint8Array || !result.isEnglish) {
    return result;
  }
  let best: AnalyseResult | null = null;
  let bestN = -1;
  for (const leaf of iterText(state)) {
    const det = leafNonEnglish(leaf);
    if (!det) continue;
    const n = alphaCount(cutCodePoints(leaf, 4000));
    if (n > bestN) {
      bestN = n;
      best = det;
    }
  }
  if (!best) return result;
  // The value pass answers `language` and `isEnglish` for the state, and leaves the script fields
  // to the state. Returning `best` whole answered `script`, `scriptProfile` and `nonLatinFraction`
  // from one leaf instead: a long English note above a short Russian message is 98.6% Latin, and
  // the result said `script: "cyrillic"` with a profile of `{cyrillic: 1}` -- a description of the
  // leaf that contradicts `scriptProfile(stateText(state))` for the same input, and that a caller
  // reading `analyse(state).script` to report the language of a ticket cannot tell from the
  // `isEnglish` flag beside it. Python's `analyse` overrides the same two fields and no others.
  return {
    ...result,
    language: best.language,
    isEnglish: false,
    languageUndecided: best.languageUndecided,
  };
}

export function isEnglish(state: unknown): boolean {
  return analyse(state).isEnglish;
}