Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

54.2 KB
Raw Blame
//! Dependency-free language/script detection used to route between Laya checkpoints. Ports
//! `laya/lang.py`.
//!
//! Routing only needs one decision: *is this English Latin text, or is it something the English
//! checkpoint cannot read?* Benchmarks on MASSIVE (14 languages) showed the English checkpoint
//! collapsing to near-random on non-Latin scripts (Hindi 0.100, Korean 0.103, Swahili 0.103, Tamil
//! 0.113 at 20 options, where random is 0.050), while holding up far better on Latin-script
//! languages (French 0.487, Spanish 0.480). So the signal that matters most is *script*, and the
//! secondary signal is whether Latin text is English.
//!
//! Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic
//! and is explicitly best-effort: pass an explicit model or `lang=` when the language is already
//! known.
//!
//! ## Unicode fidelity
//!
//! Python's `str.isalpha()` tests Unicode general category `L*` (`Lu`, `Ll`, `Lt`, `Lm`, `Lo`)
//! only. Rust's `char::is_alphabetic` also accepts combining marks, which would change
//! [`script_profile`]'s fractions on marked scripts (Hindi matras, for example), so every
//! alphabetic test here goes through [`unicode_general_category::get_general_category`] instead.
//! Python's `[^\W\d_]+` word pattern (`\w` = `isalnum()` + `_`, in Unicode mode) is likewise
//! reproduced with a general-category check rather than a regex crate: `\w` minus digits (`\d` =
//! category `Nd` only) minus `_` leaves letters plus `Nl`/`No` (a numeric-letter or other-number
//! character, but not a plain decimal digit). The other Python patterns this module uses
//! (`_IDENTIFIER`, `_CODE_LINE`, `_JOINED`, `_LETTER_RUN`) and `unicodedata.combining` are
//! hand-written for the same reason, so they agree with Python on every character rather than
//! with the `regex` crate's definition of `\w`.

use std::collections::{HashMap, HashSet};
use std::sync::LazyLock;
use unicode_general_category::{GeneralCategory, get_general_category};
use unicode_normalization::char::canonical_combining_class;

/// Namespace for language/script detection. See the module docs for the source this ports.
pub struct LanguageDetection;

/// The full detection result for one piece of state. See [`LanguageDetection::analyse`].
///
/// Marked `#[non_exhaustive]` so a field Python adds later does not break callers again.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct LanguageAnalysis {
    /// The dominant script: `"latin"`, `"han"`, `"devanagari"`, ... or `"unknown"` when the text
    /// has no letters at all.
    pub script: String,
    /// Fraction of alphabetic characters belonging to each detected script.
    pub script_profile: HashMap<String, f64>,
    /// Best-effort Latin-script language guess (an ISO-639-1-ish code), or `None` when undecided
    /// or when the script is not Latin.
    pub language: Option<String>,
    /// Whether the English checkpoint can be expected to read this text.
    pub is_english: bool,
    /// Whether [`Self::language`] is `None` because no language could be identified (as opposed
    /// to being `None` because the script itself is non-Latin).
    pub language_undecided: bool,
    /// Fraction of (lowercased) characters that are non-English Latin diacritics.
    pub diacritic_rate: f64,
    /// `1.0 - script_profile["latin"]`, or `0.0` when there is no script profile at all.
    pub non_latin_fraction: f64,
    /// The line or field the segment scan singled out as non-English inside a state that reads
    /// English as a whole, else `None`. A state flipped non-English by the per-value read that
    /// follows the segment scan is reported with this still `None`, exactly as Python does.
    pub mixed_segment: Option<String>,
}

// Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
// Order matters: `detect_script`/`script_profile` take the first matching range, and a tie
// between two script counts in `detect_script` is broken by which name was inserted first, which
// is this table's order (for non-Latin scripts) followed by "latin" last.
const SCRIPT_RANGES: &[(&str, &[(u32, u32)])] = &[
    ("greek", &[(0x0370, 0x03FF), (0x1F00, 0x1FFF)]),
    (
        "cyrillic",
        &[(0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F)],
    ),
    ("armenian", &[(0x0530, 0x058F)]),
    ("hebrew", &[(0x0590, 0x05FF)]),
    (
        "arabic",
        &[
            (0x0600, 0x06FF),
            (0x0750, 0x077F),
            (0x08A0, 0x08FF),
            (0xFB50, 0xFDFF),
            (0xFE70, 0xFEFF),
        ],
    ),
    ("devanagari", &[(0x0900, 0x097F), (0xA8E0, 0xA8FF)]),
    ("bengali", &[(0x0980, 0x09FF)]),
    ("gurmukhi", &[(0x0A00, 0x0A7F)]),
    ("gujarati", &[(0x0A80, 0x0AFF)]),
    ("oriya", &[(0x0B00, 0x0B7F)]),
    ("tamil", &[(0x0B80, 0x0BFF)]),
    ("telugu", &[(0x0C00, 0x0C7F)]),
    ("kannada", &[(0x0C80, 0x0CFF)]),
    ("malayalam", &[(0x0D00, 0x0D7F)]),
    ("sinhala", &[(0x0D80, 0x0DFF)]),
    ("thai", &[(0x0E00, 0x0E7F)]),
    ("lao", &[(0x0E80, 0x0EFF)]),
    ("tibetan", &[(0x0F00, 0x0FFF)]),
    ("myanmar", &[(0x1000, 0x109F)]),
    ("georgian", &[(0x10A0, 0x10FF)]),
    ("ethiopic", &[(0x1200, 0x137F)]),
    ("khmer", &[(0x1780, 0x17FF)]),
    (
        "hangul",
        &[(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)],
    ),
    (
        "kana",
        &[(0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF)],
    ),
    (
        "han",
        &[(0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF)],
    ),
];

/// A diacritic rate above this is taken as evidence the text is not English, even when no
/// stopword list matches it.
const NON_EN_DIACRITIC_RATE: f64 = 0.02;

/// One accented loanword or proper noun (`café`, `résumé`, `José`) must not alone pull otherwise
/// plain English off the English checkpoint: the rate is measured over every character, so a
/// single `é` in a short sentence clears the floor above. English function words keep their say
/// only through the rescue in [`english_rescued_by_words`]; a rate this high vetoes it.
const ENGLISH_RESCUE_DIACRITIC_RATE: f64 = 0.06;

/// Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a
/// brand name or order code outvotes the CJK request around it letter for letter. A short
/// message needs a large share to count; a long payload dilutes the share, so there a sentence's
/// worth of letters counts too.
const NON_LATIN_FRACTION: f64 = 0.2;
const NON_LATIN_MIN_FRACTION: f64 = 0.1;
const NON_LATIN_MIN_LETTERS: i64 = 10;

/// An all-caps line must show a matched stopword at least this long before it is believed: the
/// shouted bar of `lang.py` (`_SHOUTED_MIN_STOPWORD`), chosen against three measured corpora.
const SHOUTED_MIN_STOPWORD: usize = 4;
/// ...and a token at least this long (`_SHOUTED_MIN_WORD`): a word rather than an acronym. Also
/// decides whether an all-caps run inside mixed-case text is an acronym worth blanking.
const SHOUTED_MIN_WORD: usize = 5;

/// Letters that ordinary English does not use. This catches a Latin-script language with no
/// stopword list at all (Romanian, Polish, Czech, Turkish, Baltic, ...), which is the difference
/// between routing it to the multilingual checkpoint and silently handing it to the English one.
/// Text is lowercased before matching, so only lowercase forms need listing here.
const NON_EN_DIACRITICS: &str = concat!(
    "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ", // Western European
    "ăâîșțşţ",                         // Romanian
    "ąćęłńśźż",                        // Polish
    "čďěňřšťůž",                       // Czech / Slovak
    "őű",                              // Hungarian
    "ğı",                              // Turkish
    "āēģīķļņūž",                       // Baltic
    "đ",                               // Serbo-Croatian / Vietnamese
    "ə",                               // Azerbaijani
    // Vietnamese: no stopword list of its own, so its letters are the only signal that keeps it
    // off the English checkpoint. The set used to stop at `đ` (the comment above already claimed
    // Vietnamese coverage), but the tone-marked letters are precomposed codepoints of their own,
    // so everyday sentences such as "Cảm ơn bạn nhiều nhé" carried a diacritic rate of 0.0 and
    // were routed to the English checkpoint. Mirrors `laya/lang.py`; text is lowercased before
    // matching, so lowercase forms are enough.
    "ạảấầẩẫậắằẳẵặ",                    // a-family tone marks
    "ẹẻẽếềểễệỉị",                      // e-/i-family tone marks
    "ọỏốồổỗộớờởỡợụủứừửữự",             // o-/u-family tone marks
    "ỳỵỷỹơư",                          // y-family tone marks, and the two letters no other
                                       // listed language uses
);

// Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is
// weighted and a margin is required before calling something non-English. Generated from the
// `_STOP` sets of `laya/lang.py` (sets, so order within a list carries no meaning; the order of
// the lists, in `TABLES`, does).
//
// The Romance lists (fr/es/pt/it) deliberately carry the *unaccented* function words as well as
// the accented ones: a state that lost its accents keeps no diacritic rate for the non-English
// signal to read. `de`/`en` stay out of Spanish and `no` out of Portuguese because they are
// common English tokens. The Swedish list holds words that survive ASCII-normalising ticket
// pipelines, the Bangla list is romanized ("Banglish") and leaves out words that are also
// English, and the Azerbaijani list leaves out `her`, `ne`, `men`, `de` and `ki` (collisions).
const EN_STOP: &[&str] = &[
    "after", "already", "always", "and", "any", "are", "aren", "as", "at", "be", "because", "been",
    "before", "being", "between", "both", "but", "can", "could", "couldn", "didn", "didnt", "does",
    "doesn", "doesnt", "doing", "done", "dont", "down", "during", "each", "ever", "every", "few",
    "for", "from", "has", "hasn", "have", "haven", "havent", "here", "him", "his", "how", "i",
    "if", "in", "into", "is", "isn", "isnt", "it", "ive", "many", "might", "much", "must", "never",
    "nor", "not", "of", "on", "only", "onto", "our", "please", "shall", "she", "should", "some",
    "than", "that", "thats", "the", "their", "them", "then", "there", "these", "they", "this",
    "those", "through", "to", "too", "until", "very", "was", "wasn", "wasnt", "we", "were",
    "weren", "what", "whats", "when", "where", "which", "while", "who", "whom", "why", "will",
    "with", "within", "without", "won", "wont", "would", "wouldn", "yet", "you", "your", "youre",
    "yours",
];

const FR_STOP: &[&str] = &[
    "alors", "au", "aux", "avec", "bien", "bonjour", "ce", "ces", "cette", "comment", "dans",
    "des", "deux", "dois", "doit", "donc", "du", "elle", "elles", "est", "et", "fait", "fois",
    "il", "ils", "je", "jour", "jours", "la", "le", "les", "ma", "mais", "merci", "mes", "mois",
    "mon", "nous", "ont", "ou", "pas", "peut", "peux", "plus", "pour", "pourquoi", "quand", "que",
    "qui", "sa", "ses", "sont", "sur", "ta", "tes", "ton", "tous", "tout", "toute", "trois",
    "très", "tu", "une", "veut", "veux", "vous", "être",
];

const DE_STOP: &[&str] = &[
    "aber", "auch", "auf", "aus", "bei", "bitte", "das", "dem", "den", "der", "dich", "die",
    "diese", "diesen", "dieser", "dieses", "dir", "ein", "eine", "einem", "einen", "einer", "für",
    "gibt", "habe", "haben", "heute", "ich", "im", "in", "ist", "jetzt", "kann", "kannst", "mein",
    "meine", "meinem", "meinen", "meiner", "mich", "mir", "mit", "nach", "nicht", "noch", "oder",
    "sich", "sind", "und", "uns", "von", "wann", "was", "welche", "werden", "wie", "wir", "wird",
    "wo", "wurde", "zu", "zum", "zur",
];

const ES_STOP: &[&str] = &[
    "al", "algo", "aquí", "aunque", "como", "con", "cuando", "del", "donde", "dos", "el", "entre",
    "es", "esa", "ese", "eso", "esta", "este", "esto", "está", "fue", "fueron", "gracias", "han",
    "hay", "hemos", "hoy", "la", "las", "le", "les", "lo", "los", "mi", "muy", "más", "nada",
    "necesito", "ni", "nos", "para", "pero", "por", "porque", "puede", "pueden", "que", "quiero",
    "se", "ser", "sobre", "son", "su", "sus", "también", "tengo", "tiene", "tienen", "todo",
    "tres", "tu", "un", "una", "y", "ya",
];

const PT_STOP: &[&str] = &[
    "agora", "ainda", "alguem", "alguém", "ali", "antes", "ao", "aos", "aqui", "as", "até", "boa",
    "cadê", "com", "como", "consigo", "da", "das", "depois", "deu", "do", "dois", "dos", "e", "em",
    "entao", "então", "era", "esta", "estamos", "estava", "este", "estou", "está", "eu", "ficou",
    "fiz", "foi", "gostaria", "hoje", "isso", "isto", "ja", "já", "mais", "mas", "meu", "meus",
    "minha", "minhas", "muito", "na", "nada", "nao", "nas", "nenhum", "nenhuma", "ninguem",
    "ninguém", "noite", "nos", "nossa", "nosso", "não", "o", "obrigada", "obrigado", "olá", "onde",
    "ontem", "os", "para", "pela", "pelo", "pode", "podem", "por", "porque", "pra", "preciso",
    "quando", "que", "quero", "sao", "se", "ser", "seu", "sou", "sua", "são", "tambem", "também",
    "tarde", "tem", "tenho", "três", "tudo", "tá", "um", "uma", "vc", "vcs", "voce", "voces",
    "você", "vocês", "é",
];

const IT_STOP: &[&str] = &[
    "abbiamo", "adesso", "agli", "alla", "alle", "anche", "ancora", "avete", "che", "ci", "ciao",
    "col", "come", "con", "da", "dagli", "dal", "dalla", "dallo", "degli", "dei", "del", "della",
    "delle", "dello", "deve", "devo", "devono", "di", "dove", "e", "ed", "era", "fra", "già",
    "gli", "grazie", "ha", "hai", "hanno", "ho", "ieri", "il", "la", "le", "lo", "mai", "mi",
    "mia", "mio", "molto", "ne", "negli", "nel", "nell", "nella", "non", "o", "oggi", "per",
    "perche", "più", "poco", "quando", "questa", "questo", "scusa", "sempre", "si", "sono",
    "stata", "stato", "su", "sua", "sul", "sulla", "sulle", "tra", "tuo", "un", "una", "uno",
    "voglio", "vorrei", "è",
];

const NL_STOP: &[&str] = &[
    "aan", "dat", "deze", "door", "een", "het", "is", "maar", "met", "naar", "niet", "ook", "op",
    "te", "van", "voor", "worden", "wordt", "zijn",
];

const SV_STOP: &[&str] = &[
    "appen",
    "aterbetala",
    "aterbetalning",
    "att",
    "behover",
    "behöver",
    "blivit",
    "debiterade",
    "dessa",
    "detta",
    "faktura",
    "fel",
    "från",
    "får",
    "ganger",
    "gång",
    "gånger",
    "gör",
    "göra",
    "hittar",
    "hjalp",
    "hjalpa",
    "hjälp",
    "hjälpa",
    "här",
    "installningarna",
    "installningen",
    "inställningarna",
    "inställningen",
    "inte",
    "jag",
    "kommer",
    "kraschar",
    "kvittot",
    "losenord",
    "lösenord",
    "mig",
    "min",
    "mitt",
    "måste",
    "nar",
    "när",
    "nästa",
    "och",
    "också",
    "om",
    "oppnar",
    "paket",
    "ska",
    "skicka",
    "skulle",
    "sparningen",
    "spårningen",
    "säger",
    "till",
    "tva",
    "två",
    "upp",
    "uppdaterats",
    "utan",
    "vi",
    "vill",
    "är",
    "återbetala",
    "återbetalning",
    "öppnar",
];

const RO_STOP: &[&str] = &[
    "aceasta", "această", "acest", "acesta", "acum", "ale", "care", "dar", "din", "după", "este",
    "foarte", "fost", "fără", "lui", "mi", "nu", "pentru", "până", "sunt", "să", "trebuie",
    "vreau", "vă", "în", "și", "ți",
];

const BN_STOP: &[&str] = &[
    "abar",
    "ajke",
    "akhon",
    "amader",
    "amake",
    "amar",
    "ami",
    "amra",
    "apnake",
    "apnar",
    "apnara",
    "apni",
    "ar",
    "asbe",
    "bhai",
    "bhalo",
    "bolte",
    "bolun",
    "chai",
    "chaina",
    "dhonnobad",
    "dilam",
    "dite",
    "diye",
    "diyechi",
    "dorkar",
    "duibar",
    "ei",
    "eita",
    "ekbar",
    "ekhon",
    "ekhono",
    "ekta",
    "ferot",
    "geche",
    "gese",
    "hobe",
    "hocche",
    "hoise",
    "hoye",
    "hoyeche",
    "hoyni",
    "jabe",
    "jodi",
    "jonno",
    "kalke",
    "keno",
    "keu",
    "kharap",
    "khub",
    "ki",
    "kibhabe",
    "kichu",
    "kintu",
    "kivabe",
    "kobe",
    "kokhon",
    "korbo",
    "korchi",
    "kore",
    "koreche",
    "korechi",
    "koren",
    "korlam",
    "korsi",
    "korte",
    "korun",
    "kothay",
    "koto",
    "lagbe",
    "moddhe",
    "nai",
    "niye",
    "oi",
    "oita",
    "onek",
    "ota",
    "pabo",
    "paini",
    "parben",
    "parbo",
    "parchi",
    "parchina",
    "peyechi",
    "sathe",
    "shathe",
    "shob",
    "shomossa",
    "somossa",
    "tader",
    "tahole",
    "taka",
    "theke",
    "tomake",
    "tomar",
    "tomra",
    "tumi",
    "valo",
];

const AZ_STOP: &[&str] = &[
    "amma", "ancaq", "artiq", "artıq", "bir", "biz", "bu", "cox", "daha", "deyil", "də", "eger",
    "görə", "həm", "hər", "ile", "ilə", "isə", "kimi", "lakin", "mən", "nə", "olan", "olmasa",
    "olub", "onlar", "siz", "sonra", "sən", "ucun", "var", "ve", "və", "yalniz", "yalnız", "yox",
    "yoxdur", "çox", "üçün", "əgər",
];

const SHORT_SWEDISH_WORDS: &[&str] = &[
    "abonnemanget",
    "aterbetalning",
    "atkomst",
    "behover",
    "behöver",
    "betalningen",
    "fakturan",
    "felmeddelande",
    "glomt",
    "glömt",
    "inloggningen",
    "installningen",
    "inställningen",
    "kvitto",
    "kvittot",
    "losenord",
    "lösenord",
    "sparningen",
    "spårningen",
    "återbetalning",
    "åtkomst",
];

const NORDIC_OVERLAP_WORDS: &[&str] = &[
    "får", "hej", "ja", "jo", "kommer", "mig", "min", "nej", "om", "skulle", "tack", "vi",
];

/// The stop lists in `_STOP` iteration order (`en` first): the order is load-bearing for
/// `latin_profile`'s tie-break, which keeps the earlier language.
struct StopTables {
    lists: Vec<(&'static str, HashSet<&'static str>)>,
    /// Words that more than one list claims (plus the Swedish-Danish overlap words): a shared
    /// word may not name a winner by itself, though it still counts toward the total of a
    /// language that also matched a word of its own.
    shared: HashSet<&'static str>,
    /// English function words no other list holds. They alone carry the English rescue.
    en_only: HashSet<&'static str>,
    /// Function words of another language that are also ordinary English words: a repeat is
    /// evidence of the foreign language exactly once.
    collision: HashSet<&'static str>,
    nordic_overlap: HashSet<&'static str>,
    short_swedish: HashSet<&'static str>,
}

static TABLES: LazyLock<StopTables> = LazyLock::new(|| {
    let raw: [(&'static str, &'static [&'static str]); 11] = [
        ("en", EN_STOP),
        ("fr", FR_STOP),
        ("de", DE_STOP),
        ("es", ES_STOP),
        ("pt", PT_STOP),
        ("it", IT_STOP),
        ("nl", NL_STOP),
        ("sv", SV_STOP),
        ("ro", RO_STOP),
        ("bn", BN_STOP),
        ("az", AZ_STOP),
    ];
    let lists: Vec<(&'static str, HashSet<&'static str>)> = raw
        .iter()
        .map(|(code, words)| (*code, words.iter().copied().collect()))
        .collect();
    let mut membership: HashMap<&'static str, usize> = HashMap::new();
    for (_, set) in &lists {
        for word in set {
            *membership.entry(word).or_insert(0) += 1;
        }
    }
    let nordic_overlap: HashSet<&'static str> = NORDIC_OVERLAP_WORDS.iter().copied().collect();
    let mut shared: HashSet<&'static str> = membership
        .into_iter()
        .filter(|&(_, n)| n > 1)
        .map(|(w, _)| w)
        .collect();
    shared.extend(nordic_overlap.iter().copied());
    let en_only = lists[0].1.difference(&shared).copied().collect();
    let foreign: HashSet<&'static str> = lists[1..]
        .iter()
        .flat_map(|(_, set)| set.iter().copied())
        .collect();
    let collision = ["come", "son", "do", "care", "todo", "im", "per", "plus"]
        .into_iter()
        .filter(|w| foreign.contains(w))
        .collect();
    StopTables {
        lists,
        shared,
        en_only,
        collision,
        nordic_overlap,
        short_swedish: SHORT_SWEDISH_WORDS.iter().copied().collect(),
    }
});

fn stop_set(code: &str) -> Option<&'static HashSet<&'static str>> {
    TABLES
        .lists
        .iter()
        .find(|(c, _)| *c == code)
        .map(|(_, set)| set)
}

// ---------------------------------------------------------------------------------------------
// Python-compatible character classes. `regex`'s `\w`, `\s` and `\b` differ from `re`'s on marks,
// joiners and a few control characters, so the patterns `lang.py` runs are hand-written over
// these predicates.
// ---------------------------------------------------------------------------------------------

/// `str.isspace()`: Unicode White_Space plus U+001C..U+001F (bidi class B/S), which `re`'s `\s`
/// accepts and the `regex` crate's does not.
pub(crate) fn py_is_space(ch: char) -> bool {
    ch.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&ch)
}

/// `str.strip()`.
pub(crate) fn py_strip(s: &str) -> &str {
    s.trim_matches(py_is_space)
}

/// `\d`: decimal digits (`Nd`) only.
fn is_decimal(ch: char) -> bool {
    get_general_category(ch) == GeneralCategory::DecimalNumber
}

/// `[^\W_]`: `isalnum()` -- letters and every kind of number, but not the underscore.
fn is_alnum(ch: char) -> bool {
    LanguageDetection::is_alpha(ch)
        || matches!(
            get_general_category(ch),
            GeneralCategory::DecimalNumber
                | GeneralCategory::LetterNumber
                | GeneralCategory::OtherNumber
        )
}

/// `\w`: [`is_alnum`] plus the underscore.
pub(crate) fn is_word_char(ch: char) -> bool {
    ch == '_' || is_alnum(ch)
}

/// `[^\W\d_]`: a letter, or a numeral that is not a plain decimal digit.
pub(crate) fn is_letter_word(ch: char) -> bool {
    is_alnum(ch) && !is_decimal(ch)
}

fn is_titlecase(ch: char) -> bool {
    get_general_category(ch) == GeneralCategory::TitlecaseLetter
}

/// `ch.isupper()` for a one-character string.
pub(crate) fn char_is_upper(ch: char) -> bool {
    ch.is_uppercase() && !ch.is_lowercase() && !is_titlecase(ch)
}

/// `ch.islower()` for a one-character string.
fn char_is_lower(ch: char) -> bool {
    ch.is_lowercase() && !ch.is_uppercase() && !is_titlecase(ch)
}

/// `s.isupper()`: at least one uppercase letter and no lowercase or titlecase one.
fn str_is_upper(s: &str) -> bool {
    let mut cased = false;
    for ch in s.chars() {
        if ch.is_lowercase() || is_titlecase(ch) {
            return false;
        }
        if ch.is_uppercase() {
            cased = true;
        }
    }
    cased
}

/// The first `n` characters of `s` (Python's `s[:n]`).
pub(crate) fn prefix_chars(s: &str, n: usize) -> &str {
    match s.char_indices().nth(n) {
        Some((idx, _)) => &s[..idx],
        None => s,
    }
}

/// Maximal runs of `[^\W\d_]` in `text` (`_WORD.findall`).
fn word_runs(text: &str) -> Vec<&str> {
    let mut words = Vec::new();
    let mut start: Option<usize> = None;
    for (idx, ch) in text.char_indices() {
        if is_letter_word(ch) {
            if start.is_none() {
                start = Some(idx);
            }
        } else if let Some(s) = start.take() {
            words.push(&text[s..idx]);
        }
    }
    if let Some(s) = start {
        words.push(&text[s..]);
    }
    words
}

/// `_IDENTIFIER.sub(" ", text)`: `(?<![\w-])[\w-]*(?:[.@][\w-]+)+`. A token whose dot or `@` joins
/// word characters is an identifier, not prose (`github.com`, `[email protected]`, `v1.2.3`). A
/// sentence-final period keeps its word: the pattern needs word characters on both sides.
fn replace_identifiers(text: &str) -> String {
    let is_ident = |c: char| is_word_char(c) || c == '-';
    let chars: Vec<char> = text.chars().collect();
    let n = chars.len();
    let mut out = String::with_capacity(text.len());
    let mut i = 0;
    while i < n {
        // The lookbehind only lets a match start at the head of a run, which is also what keeps
        // this linear: a shorter `[\w-]*` could never be followed by `[.@]`.
        if i == 0 || !is_ident(chars[i - 1]) {
            let mut j = i;
            while j < n && is_ident(chars[j]) {
                j += 1;
            }
            let mut groups = 0;
            while j < n && (chars[j] == '.' || chars[j] == '@') {
                let mut k = j + 1;
                while k < n && is_ident(chars[k]) {
                    k += 1;
                }
                if k == j + 1 {
                    break;
                }
                j = k;
                groups += 1;
            }
            if groups > 0 {
                out.push(' ');
                i = j;
                continue;
            }
        }
        out.push(chars[i]);
        i += 1;
    }
    out
}

/// `_CODE_LINE.search`: `[=;{}\[\]]|\w\(` -- a line carrying code syntax is not prose.
fn has_code_syntax(line: &str) -> bool {
    let mut prev_word = false;
    for ch in line.chars() {
        if matches!(ch, '=' | ';' | '{' | '}' | '[' | ']') || (ch == '(' && prev_word) {
            return true;
        }
        prev_word = is_word_char(ch);
    }
    false
}

/// `_JOINED.search(token)`: `[^\W_][._/\\][^\W_]` -- a name, path or compound (`Nav/Com`, `OS/2`,
/// `C:\DOS\mode`), not a sentence.
fn is_joined_token(token: &str) -> bool {
    let chars: Vec<char> = token.chars().collect();
    chars
        .windows(3)
        .any(|w| is_alnum(w[0]) && matches!(w[1], '.' | '_' | '/' | '\\') && is_alnum(w[2]))
}

/// `_LETTER_RUN.finditer`: maximal runs of at least two `[^\W\d_]` characters, as byte ranges.
fn letter_runs(text: &str) -> Vec<(usize, usize)> {
    let mut runs = Vec::new();
    let mut start: Option<(usize, usize)> = None; // (byte offset, char count so far)
    for (idx, ch) in text.char_indices() {
        if is_letter_word(ch) {
            start = Some(match start {
                Some((s, n)) => (s, n + 1),
                None => (idx, 1),
            });
        } else if let Some((s, n)) = start.take()
            && n >= 2
        {
            runs.push((s, idx));
        }
    }
    if let Some((s, n)) = start
        && n >= 2
    {
        runs.push((s, text.len()));
    }
    runs
}

/// `_LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), text)`.
fn blank_upper_runs(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    let mut last = 0;
    for (s, e) in letter_runs(text) {
        if str_is_upper(&text[s..e]) {
            out.push_str(&text[last..s]);
            out.push(' ');
            last = e;
        }
    }
    out.push_str(&text[last..]);
    out
}

impl LanguageDetection {
    /// Whether `ch` is alphabetic under Python's `str.isalpha()` rule: Unicode general category
    /// `Lu`, `Ll`, `Lt`, `Lm` or `Lo`. Unlike `char::is_alphabetic`, this excludes combining
    /// marks.
    pub(crate) fn is_alpha(ch: char) -> bool {
        matches!(
            get_general_category(ch),
            GeneralCategory::UppercaseLetter
                | GeneralCategory::LowercaseLetter
                | GeneralCategory::TitlecaseLetter
                | GeneralCategory::ModifierLetter
                | GeneralCategory::OtherLetter
        )
    }

    /// First script whose range table contains `cp`, if any.
    fn script_for_codepoint(cp: u32) -> Option<&'static str> {
        for (name, ranges) in SCRIPT_RANGES {
            if ranges.iter().any(|&(lo, hi)| cp >= lo && cp <= hi) {
                return Some(name);
            }
        }
        None
    }

    /// Whether `cp` counts as Latin: Basic Latin through the IPA Extensions (below U+02B0),
    /// Latin Extended Additional, and the fullwidth Latin letters.
    fn is_latin_codepoint(cp: u32) -> bool {
        cp < 0x02B0
            || (0x1E00..=0x1EFF).contains(&cp)
            || (0xFF21..=0xFF3A).contains(&cp)
            || (0xFF41..=0xFF5A).contains(&cp)
    }

    /// Collect the string leaves of a state (str / object / array), so detection sees real
    /// content. Depths beyond 6 are dropped, and keys are ignored (they are usually English).
    fn iter_text<'a>(state: &'a serde_json::Value, depth: u32, out: &mut Vec<&'a str>) {
        if depth > 6 {
            return;
        }
        match state {
            serde_json::Value::String(s) => out.push(s),
            serde_json::Value::Object(map) => {
                for v in map.values() {
                    Self::iter_text(v, depth + 1, out);
                }
            }
            serde_json::Value::Array(items) => {
                for v in items {
                    Self::iter_text(v, depth + 1, out);
                }
            }
            _ => {}
        }
    }

    fn leaves(state: &serde_json::Value) -> Vec<&str> {
        let mut out = Vec::new();
        Self::iter_text(state, 0, &mut out);
        out
    }

    /// Flatten a state into the text used for detection (keys are ignored: they are usually
    /// English). `max_chars` truncates by Unicode scalar value, matching Python's `[:max_chars]`
    /// string slice.
    pub fn state_text(state: &serde_json::Value, max_chars: usize) -> String {
        // lang.py:state_text stops adding leaves once the budget (which counts each joining
        // space) is spent, so a leaf that ends exactly at the budget gets no trailing space.
        let mut parts: Vec<&str> = Vec::new();
        let mut budget = max_chars as i64;
        for leaf in Self::leaves(state) {
            if budget <= 0 {
                break;
            }
            let len = leaf.chars().count() as i64;
            if len > budget {
                parts.push(prefix_chars(leaf, budget as usize));
                break;
            }
            parts.push(leaf);
            budget -= len + 1;
        }
        prefix_chars(&parts.join(" "), max_chars).to_string()
    }

    /// [`Self::state_text`] with Python's default `max_chars=4000`.
    pub fn state_text_default(state: &serde_json::Value) -> String {
        Self::state_text(state, 4000)
    }

    /// The state Python's `_iter_text` sees for a `bytes`/`bytearray` state: the bytes decoded as
    /// strict UTF-8, or no text at all when they are not valid UTF-8 (`lang.py:253-255`).
    fn bytes_state(bytes: &[u8]) -> serde_json::Value {
        serde_json::Value::String(std::str::from_utf8(bytes).unwrap_or("").to_string())
    }

    /// [`Self::state_text`] for a `bytes` state, as Python accepts one: strict UTF-8, and invalid
    /// UTF-8 reads as no text (`""`).
    pub fn state_text_bytes(state: &[u8], max_chars: usize) -> String {
        Self::state_text(&Self::bytes_state(state), max_chars)
    }

    /// [`Self::analyse`] for a `bytes` state, as Python accepts one: strict UTF-8, and invalid
    /// UTF-8 reads as no text.
    pub fn analyse_bytes(state: &[u8]) -> LanguageAnalysis {
        Self::analyse(&Self::bytes_state(state))
    }

    /// [`Self::is_english`] for a `bytes` state.
    pub fn is_english_bytes(state: &[u8]) -> bool {
        Self::analyse_bytes(state).is_english
    }

    /// Dominant script of `text`: `"latin"`, `"han"`, `"devanagari"`, ... or `"unknown"` if there
    /// are no letters.
    ///
    /// Ties are broken by first-inserted order: the non-Latin scripts in the order they first
    /// appear in `text`, then `"latin"` last, so Latin never wins a tie against a non-Latin
    /// script present in the same text.
    pub fn detect_script(text: &str) -> String {
        ScriptCounts::of(text).dominant().to_string()
    }

    /// Fraction of alphabetic characters belonging to each detected script.
    pub fn script_profile(text: &str) -> HashMap<String, f64> {
        ScriptCounts::of(text).profile()
    }

    /// Evidence behind the Latin-script language guess: `language` (may be `None` when
    /// undecided), `english_hits`, `diacritic_rate` and `looks_non_english`. [`Self::analyse`]
    /// needs the evidence and not just the verdict, because "undecided" and "English" are
    /// different answers and only one of them is safe to send to the English checkpoint.
    ///
    /// A non-English language is only named when it matched at least one word that no other list
    /// claims: shared function words alone (`la`, `e`, `o`) identify no particular language.
    /// `looks_non_english` can be true for non-English diacritics or an overlapping
    /// Swedish-Danish marker even when no language is named.
    pub fn latin_profile(text: &str) -> LatinProfile {
        let p = latin_profile_inner(text);
        LatinProfile {
            language: p.language.map(str::to_string),
            english_hits: p.english_hits,
            diacritic_rate: p.diacritic_rate,
            looks_non_english: p.looks_non_english,
        }
    }

    /// Best-effort language code for Latin-script text, or `None` when undecided.
    ///
    /// Scores function-word hits per language and requires the winner to beat English by a
    /// margin and to have matched at least one word of its own, so ordinary English is never
    /// misrouted and a word of several languages at once names none of them. Short inputs
    /// usually return `None` on purpose.
    pub fn guess_latin_language(text: &str) -> Option<String> {
        Self::latin_profile(text).language
    }

    /// Full detection result for a state. See [`LanguageAnalysis`].
    ///
    /// String values are what get read. A state that reads English as a whole is then checked
    /// line by line and field by field, because the English part is usually the longer one and
    /// outvotes a short message in another language (a Portuguese ticket with an English stack
    /// trace); one non-English value is enough to move the state to the multilingual checkpoint.
    pub fn analyse(state: &serde_json::Value) -> LanguageAnalysis {
        let leaves = Self::leaves(state);
        let mut result = analyse_text(&Self::state_text_default(state));
        // a single line has no other part to be outvoted by, and was just read whole
        if result.script == "latin"
            && result.is_english
            && (leaves.len() > 1 || leaves.iter().any(|leaf| leaf.contains('\n')))
            && let Some((lang, mixed)) = non_english_segment(&leaves, 4000)
        {
            result.language = Some(lang.to_string());
            result.is_english = false;
            result.language_undecided = false;
            result.mixed_segment = Some(mixed);
        }
        // A plain string was just read whole. A structured state can still hide a message past
        // the segment cap, or in a script `latin_profile` does not name.
        if matches!(
            state,
            serde_json::Value::String(_) | serde_json::Value::Null
        ) || !result.is_english
        {
            return result;
        }
        let mut best_n: i64 = -1;
        let mut best: Option<LanguageAnalysis> = None;
        for leaf in &leaves {
            let Some(det) = leaf_non_english(leaf) else {
                continue;
            };
            let n_alpha = prefix_chars(leaf, 4000)
                .chars()
                .filter(|&c| Self::is_alpha(c))
                .count() as i64;
            if n_alpha > best_n {
                best_n = n_alpha;
                best = Some(det);
            }
        }
        if let Some(best) = best {
            result.language = best.language;
            result.is_english = false;
            result.language_undecided = best.language_undecided;
        }
        result
    }

    /// True when the English checkpoint can be expected to read this state.
    pub fn is_english(state: &serde_json::Value) -> bool {
        Self::analyse(state).is_english
    }
}

/// The evidence [`LanguageDetection::latin_profile`] gathers for a Latin-script language guess.
#[derive(Debug, Clone, PartialEq)]
pub struct LatinProfile {
    /// The guessed language code, or `None` when undecided.
    pub language: Option<String>,
    /// How many English stopword hits the text had.
    pub english_hits: i64,
    /// Fraction of (lowercased) characters that are non-English Latin diacritics.
    pub diacritic_rate: f64,
    /// Whether the text looks non-English on its own evidence: a diacritic rate at or above the
    /// threshold, or a Swedish-Danish overlap marker with no language named.
    pub looks_non_english: bool,
}

struct LatinProfileInner {
    language: Option<&'static str>,
    english_hits: i64,
    diacritic_rate: f64,
    looks_non_english: bool,
}

/// Per-script letter counts of one text, in `lang.py`'s `_script_counts` insertion order.
struct ScriptCounts {
    /// Named scripts (and `"other"`, for letters no range claims) in first-seen order.
    named: Vec<(&'static str, i64)>,
    latin: i64,
}

impl ScriptCounts {
    fn of(text: &str) -> Self {
        let mut named: Vec<(&'static str, i64)> = Vec::new();
        let mut latin = 0;
        for ch in text.chars() {
            if !LanguageDetection::is_alpha(ch) {
                continue;
            }
            let cp = ch as u32;
            if LanguageDetection::is_latin_codepoint(cp) {
                latin += 1; // Latin, IPA Extensions, Extended Additional, fullwidth
                continue;
            }
            // A letter no range claims is counted under "other": text in an unlisted script must
            // stay visible and non-Latin, never be reported as "unknown" and handed to English.
            let name = LanguageDetection::script_for_codepoint(cp).unwrap_or("other");
            match named.iter_mut().find(|(n, _)| *n == name) {
                Some(entry) => entry.1 += 1,
                None => named.push((name, 1)),
            }
        }
        Self { named, latin }
    }

    fn total(&self) -> i64 {
        self.latin + self.named.iter().map(|(_, c)| c).sum::<i64>()
    }

    /// `_script_from_counts`: the first script with the highest count; Latin is last, so a named
    /// script wins a tie against it.
    fn dominant(&self) -> &'static str {
        if self.total() == 0 {
            return "unknown";
        }
        let mut best: Option<(&'static str, i64)> = None;
        for &(name, count) in self
            .named
            .iter()
            .chain(std::iter::once(&("latin", self.latin)))
        {
            if best.is_none_or(|(_, b)| count > b) {
                best = Some((name, count));
            }
        }
        best.map_or("unknown", |(name, _)| name)
    }

    fn profile(&self) -> HashMap<String, f64> {
        let total = self.total();
        if total == 0 {
            return HashMap::new();
        }
        let mut out = HashMap::new();
        if self.latin != 0 {
            out.insert("latin".to_string(), self.latin as f64 / total as f64);
        }
        for &(name, count) in &self.named {
            if count != 0 {
                out.insert(name.to_string(), count as f64 / total as f64);
            }
        }
        out
    }

    /// `max(s for s in prof if s != "latin", key=prof.get)`: the first non-Latin script with the
    /// highest count.
    fn dominant_non_latin(&self) -> Option<&'static str> {
        let mut best: Option<(&'static str, i64)> = None;
        for &(name, count) in &self.named {
            if best.is_none_or(|(_, b)| count > b) {
                best = Some((name, count));
            }
        }
        best.map(|(name, _)| name)
    }
}

fn round4(x: f64) -> f64 {
    crate::calibration::Calibration::round4(x)
}

/// Whether the non-Latin letters of a Latin-plurality text are enough to carry the state.
fn non_latin_carries(non_latin: f64, n_non_latin: i64) -> bool {
    non_latin >= NON_LATIN_FRACTION
        || (non_latin >= NON_LATIN_MIN_FRACTION && n_non_latin >= NON_LATIN_MIN_LETTERS)
}

/// `_script_of`: the named non-Latin script of one character, or `None` for Latin and for
/// characters no range claims. (Not restricted to letters: script punctuation joins its run.)
fn script_of(ch: char) -> Option<&'static str> {
    let cp = ch as u32;
    if cp < 0x0250
        || (0x1E00..=0x1EFF).contains(&cp)
        || (0xFF21..=0xFF3A).contains(&cp)
        || (0xFF41..=0xFF5A).contains(&cp)
    {
        return None;
    }
    LanguageDetection::script_for_codepoint(cp)
}

/// `_non_latin_words`: non-Latin runs that read as words rather than as annotation inside
/// English prose. A symbol (`Set α to 0.05`, one letter), a proper name (`Дмитрий Петрович`,
/// capitalised) and a pronunciation (`[vlɐˈdʲimʲɪr]`, which no range claims) are excluded. A
/// character with a canonical combining class belongs to the letter before it and never splits a
/// word.
fn non_latin_words(text: &str) -> Vec<String> {
    let mut runs: Vec<String> = Vec::new();
    let mut cur = String::new();
    let mut script: Option<&'static str> = None;
    for ch in text.chars() {
        if canonical_combining_class(ch) != 0 {
            continue;
        }
        let s = script_of(ch);
        if s.is_some() && s == script {
            cur.push(ch);
            continue;
        }
        if !cur.is_empty() {
            runs.push(std::mem::take(&mut cur));
        }
        if s.is_some() {
            cur.push(ch);
        }
        script = s;
    }
    if !cur.is_empty() {
        runs.push(cur);
    }
    runs.into_iter()
        .filter(|w| w.chars().count() >= 2 && !w.chars().next().is_some_and(char_is_upper))
        .collect()
}

/// `_english_rescued_by_words`: whether plain-English function words outvote a marginal
/// diacritic rate. English still wins when it shows at least two distinct function words no
/// other list holds and at most one word carrying a non-English letter. A rate well above
/// [`ENGLISH_RESCUE_DIACRITIC_RATE`] vetoes regardless.
fn english_rescued_by_words(word_set: &HashSet<&str>, diac_rate: f64) -> bool {
    if diac_rate >= ENGLISH_RESCUE_DIACRITIC_RATE {
        return false;
    }
    if word_set.intersection(&TABLES.en_only).count() < 2 {
        return false;
    }
    word_set
        .iter()
        .filter(|w| w.chars().any(|c| NON_EN_DIACRITICS.contains(c)))
        .count()
        <= 1
}

fn latin_profile_inner(text: &str) -> LatinProfileInner {
    // 'İ'.lower() is 'i' + a combining dot, which matches no word list.
    let cleaned = replace_identifiers(text).replace('İ', "i").to_lowercase();
    let words = word_runs(&cleaned);
    let lowered = text.to_lowercase();
    let diac = lowered
        .chars()
        .filter(|c| NON_EN_DIACRITICS.contains(*c))
        .count();
    let diac_rate = diac as f64 / (lowered.chars().count().max(1) as f64);
    let non_english = diac_rate >= NON_EN_DIACRITIC_RATE;
    let word_set: HashSet<&str> = words.iter().copied().collect();
    let tables = &*TABLES;
    let nordic_overlap =
        !word_set.is_disjoint(&tables.nordic_overlap) && word_set.is_disjoint(&tables.en_only);
    if words.len() > 1 && words.len() < 4 && !word_set.is_disjoint(&tables.short_swedish) {
        return LatinProfileInner {
            language: Some("sv"),
            english_hits: 0,
            diacritic_rate: diac_rate,
            looks_non_english: non_english,
        };
    }
    if words.len() < 4 {
        return LatinProfileInner {
            language: None,
            english_hits: 0,
            diacritic_rate: diac_rate,
            looks_non_english: non_english || nordic_overlap,
        };
    }

    // A collision word counts once however often it repeats; every other word counts its hits.
    let mut counts: HashMap<&str, i64> = HashMap::new();
    for w in &words {
        *counts.entry(w).or_insert(0) += 1;
    }
    let score_of = |set: &HashSet<&'static str>| -> i64 {
        counts
            .iter()
            .filter(|(w, _)| set.contains(*w))
            .map(|(w, n)| if tables.collision.contains(w) { 1 } else { *n })
            .sum()
    };
    let en = score_of(&tables.lists[0].1);

    // Only a language that matched at least one word no other list claims may be named. Without
    // that the top score can be pure overlap -- `la` and `e` in Romanian text made Italian the
    // winner -- which is a guess dressed as a detection. Such a language is dropped from the
    // running rather than merely losing the tie. Ties keep the earlier language (`max()` returns
    // the first maximum in `_STOP` order).
    let mut best_lg: Option<&'static str> = None;
    let mut best: i64 = 0;
    for (code, set) in tables.lists.iter().skip(1) {
        if !word_set
            .iter()
            .any(|w| set.contains(w) && !tables.shared.contains(w))
        {
            continue;
        }
        let score = score_of(set);
        if best_lg.is_none() || score > best {
            best_lg = Some(code);
            best = score;
        }
    }

    let mut language: Option<&'static str> = None;
    if best_lg.is_some() && best >= 2i64.max(en + 2) {
        // a non-English language needs a clear margin over English function words
        language = best_lg;
    } else if best_lg == Some("sv")
        && word_set.contains("inte")
        && word_set.contains("kan")
        && matches!(words[0], "kan" | "jag" | "vi")
        && en <= 1
    {
        // Short login requests such as "kan inte logga in" carry a distinctive Swedish phrase
        // but also one English-shaped token (`in`). `kan` alone is not Swedish: it is common in
        // Danish and Norwegian too.
        language = best_lg;
    } else if best_lg.is_some() && non_english && best >= 2i64.max(en) {
        // Needs two hits here too: one shared function word ("para" in Turkish text) named
        // Spanish on the strength of the diacritics alone, which is a guess dressed as a
        // detection.
        language = best_lg;
    } else if en > 0 && (!non_english || english_rescued_by_words(&word_set, diac_rate)) {
        language = Some("en");
    }
    LatinProfileInner {
        language,
        english_hits: en,
        diacritic_rate: diac_rate,
        looks_non_english: non_english || (language.is_none() && nordic_overlap),
    }
}

/// `_shouted`: true for a *cased* segment written entirely in capitals. The `isupper` half is
/// not redundant: a caseless script (Devanagari, Bengali, CJK) has no lowercase either, and the
/// bar belongs to cased text.
fn shouted(text: &str) -> bool {
    text.chars().any(char_is_upper) && !text.chars().any(char_is_lower)
}

/// `_shouted_evidence`: whether an all-caps line's evidence for `lang` is more than
/// acronym-shaped tokens.
fn shouted_evidence(tokens: &[&str], lang: &str, diacritic_rate: f64) -> bool {
    if !tokens.iter().any(|t| t.chars().count() >= SHOUTED_MIN_WORD) {
        return false;
    }
    if diacritic_rate >= NON_EN_DIACRITIC_RATE {
        return true;
    }
    let Some(stop) = stop_set(lang) else {
        return false;
    };
    tokens
        .iter()
        .map(|t| t.to_lowercase())
        .any(|w| stop.contains(w.as_str()) && w.chars().count() >= SHOUTED_MIN_STOPWORD)
}

/// `_named_prose_language`: the language code for one non-code line, or `None` when it does not
/// name a foreign language. Four words, a language `latin_profile` will name, and two
/// *different* words of that language; acronyms and slash compounds are not words, and a segment
/// in all capitals must also clear [`shouted_evidence`].
fn named_prose_language(segment: &str) -> Option<&'static str> {
    if py_strip(segment).is_empty() || has_code_syntax(segment) {
        return None;
    }
    let mut prose = segment
        .split(py_is_space)
        .filter(|tok| !tok.is_empty() && !is_joined_token(tok))
        .collect::<Vec<_>>()
        .join(" ");
    let is_shouted = shouted(&prose);
    if !is_shouted {
        prose = blank_upper_runs(&prose);
    }
    let tokens = word_runs(&prose);
    if tokens.len() < 4 {
        return None;
    }
    let prof = latin_profile_inner(&prose);
    let lang = prof.language?;
    if lang == "en" {
        return None;
    }
    if is_shouted && !shouted_evidence(&tokens, lang, prof.diacritic_rate) {
        return None;
    }
    let stop = stop_set(lang)?;
    let distinct: HashSet<String> = tokens.iter().map(|t| t.to_lowercase()).collect();
    if distinct
        .iter()
        .filter(|w| stop.contains(w.as_str()))
        .count()
        < 2
    {
        return None;
    }
    Some(lang)
}

/// `_non_english_segment`: the first line or field that, read on its own, is named a non-English
/// language -- `(language, segment)`. Reads at most `max_chars` characters in all.
fn non_english_segment(leaves: &[&str], max_chars: usize) -> Option<(&'static str, String)> {
    let mut seen = 0usize;
    for leaf in leaves {
        for seg in leaf.split('\n') {
            if seen >= max_chars {
                return None;
            }
            let seg = prefix_chars(seg, max_chars - seen);
            seen += seg.chars().count();
            if let Some(lang) = named_prose_language(seg) {
                return Some((lang, py_strip(seg).to_string()));
            }
        }
    }
    None
}

/// `_analyse_text`: the detection result for one already-flattened string. Does not look for a
/// foreign line inside mostly-English text; [`LanguageDetection::analyse`] does that.
fn analyse_text(text: &str) -> LanguageAnalysis {
    let counts = ScriptCounts::of(text);
    let total = counts.total();
    let prof = counts.profile();
    let mut script = counts.dominant();
    let non_latin = if total > 0 {
        round4(1.0 - counts.latin as f64 / total as f64)
    } else {
        0.0
    };
    let n_non_latin = (non_latin * total as f64).round_ties_even() as i64;
    if script == "latin"
        && !non_latin_words(text).is_empty()
        && non_latin_carries(non_latin, n_non_latin)
        && let Some(promoted) = counts.dominant_non_latin()
    {
        script = promoted;
    }
    if script == "unknown" {
        return LanguageAnalysis {
            script: "unknown".to_string(),
            script_profile: prof,
            language: None,
            is_english: true,
            language_undecided: true,
            diacritic_rate: 0.0,
            non_latin_fraction: 0.0,
            mixed_segment: None,
        };
    }
    if script != "latin" {
        return LanguageAnalysis {
            script: script.to_string(),
            script_profile: prof,
            language: None,
            is_english: false,
            language_undecided: true,
            diacritic_rate: 0.0,
            non_latin_fraction: non_latin,
            mixed_segment: None,
        };
    }
    let prof_lat = latin_profile_inner(text);
    let mut lang = prof_lat.language;
    // The acronym bar belongs here too, not only in the segment scan: a state that is nothing
    // but an acronym line (`MON DES EST LA`) must not be named foreign outright. Vetoing leaves
    // the text undecided, so `looks_non_english` still decides it. Neither bar applies when the
    // state is carried by a caseless script: they take evidence away from the Latin letters, and
    // `looks_non_english` reads Latin diacritics only.
    if let Some(named) = lang
        && named != "en"
        && !non_latin_carries(non_latin, n_non_latin)
    {
        if shouted(text) {
            if !shouted_evidence(&word_runs(text), named, prof_lat.diacritic_rate) {
                lang = None;
            }
        } else {
            // Mixed-case text: re-take the verdict without the all-caps runs, but only while
            // every one of them is short enough to be an acronym -- a run of
            // `SHOUTED_MIN_WORD` letters is a word, and a word is not blanked out of its own
            // sentence. `looks_non_english` and `diacritic_rate` stay measured on the original
            // text.
            let caps: Vec<&str> = letter_runs(text)
                .into_iter()
                .map(|(s, e)| &text[s..e])
                .filter(|run| str_is_upper(run))
                .collect();
            if !caps.is_empty() && !caps.iter().any(|c| c.chars().count() >= SHOUTED_MIN_WORD) {
                let blanked = blank_upper_runs(text);
                if blanked != text {
                    lang = latin_profile_inner(&blanked).language;
                }
            }
        }
    }
    // Undecided is not English. Treating it as English sent every Latin-script language with no
    // stopword list here to the checkpoint that cannot read it, silently. When nothing
    // identifies the language, non-English letters or a shared Swedish-Danish marker can still
    // prefer the multilingual checkpoint; text with neither signal goes to the English one.
    let undecided = lang.is_none();
    let english = lang == Some("en") || (undecided && !prof_lat.looks_non_english);
    LanguageAnalysis {
        script: "latin".to_string(),
        script_profile: prof,
        language: lang.map(str::to_string),
        is_english: english,
        language_undecided: undecided,
        diacritic_rate: round4(prof_lat.diacritic_rate),
        non_latin_fraction: non_latin,
        mixed_segment: None,
    }
}

/// `_leaf_non_english`: a string value that is itself not safe for the English checkpoint, else
/// `None`. A one-word name and a capitalised non-Latin name stay out, as do code lines,
/// acronyms and slash compounds. Each line is capped at 4000 characters; a long earlier field
/// does not consume the budget of the next one.
fn leaf_non_english(leaf: &str) -> Option<LanguageAnalysis> {
    let mut best_n: i64 = -1;
    let mut best: Option<LanguageAnalysis> = None;
    for line in leaf.split('\n') {
        // A line this short cannot be selected: each branch below needs four words (three
        // separators, seven characters) or `NON_LATIN_MIN_LETTERS` letters.
        if line.chars().count() < 7 {
            continue;
        }
        let sample = prefix_chars(line, 4000);
        if py_strip(sample).is_empty() || has_code_syntax(sample) {
            continue;
        }
        let det = analyse_text(sample);
        if det.is_english {
            continue;
        }
        // A named language still has to survive `named_prose_language`, but a veto is not a
        // verdict of English: a line carrying non-English letters that no stopword list can
        // name falls through to the diacritic branch.
        let named = det.language.as_deref().is_some_and(|l| l != "en")
            && named_prose_language(sample).is_some();
        let n_alpha = sample
            .chars()
            .filter(|&c| LanguageDetection::is_alpha(c))
            .count() as i64;
        if !named {
            if det.script != "latin" && det.script != "unknown" {
                if non_latin_words(sample).is_empty() || n_alpha < NON_LATIN_MIN_LETTERS {
                    continue;
                }
            } else if !(det.diacritic_rate >= NON_EN_DIACRITIC_RATE && word_runs(sample).len() >= 4)
            {
                continue;
            }
        }
        if n_alpha > best_n {
            best_n = n_alpha;
            best = Some(det);
        }
    }
    best
}