Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): route plain-ASCII Romance text to the multilingual checkpoint (#178)

devloper961-maker417089720510
diff --git a/laya/lang.py b/laya/lang.py
index 0d1ed7bc..a030fafc 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -44,20 +44,57 @@ _SCRIPT_RANGES = [
# Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is
# weighted and a margin is required before calling something non-English.
+#
+# The Romance lists (fr/es/pt/it) deliberately carry the *unaccented* function words as well as the
+# accented ones. A state that lost its accents -- mail clients, ticket systems and any pipeline that
+# normalises to ASCII strip them -- keeps no diacritic rate for the non-English signal to read, so
+# `la`, `un`, `y`, `e`, `et`, `deux` and friends are the only evidence left. With a list of mostly
+# accented words such a state produced one hit or none, fell under the two-hit margin below, and was
+# handed to the English checkpoint as undecided-but-not-non-English text (see #172 and #54).
_STOP = {
"en": {"the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
"this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
"will", "can", "would", "there", "their", "what", "which", "please", "we", "i"},
"fr": {"le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
- "pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce"},
+ "pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce",
+ "et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",
+ "ma", "ta", "sa", "mes", "tes", "ses", "ces", "deux", "trois", "très", "bien",
+ "tout", "tous", "toute", "fait", "veux", "veut", "peux", "peut", "dois", "doit",
+ "merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi",
+ "alors", "donc"},
"de": {"der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
"auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber"},
"es": {"el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
- "pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus"},
+ "pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",
+ # `de`/`en` are Spanish too, but they are common English tokens as well (`de facto`,
+ # `en-US`, `en route`, `Rio de Janeiro`), and a state of those alone already carries
+ # no English function word for the margin below to weigh them against, so they stay out.
+ "la", "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos",
+ "ni", "dos", "tres", "fue", "fueron", "ser", "tiene", "tienen", "tengo", "puede",
+ "pueden", "quiero", "necesito", "hemos", "han", "sobre", "entre", "cuando", "donde",
+ "porque", "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada", "algo",
+ "aquí", "hoy", "gracias"},
"pt": {"os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
- "dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela"},
+ "dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela",
+ # `no` is Portuguese too, and among the most frequent words it has; it is also one of the
+ # most frequent English words, so it stays out and short Portuguese states that lean on it
+ # alone are left to the diacritic rate, as before.
+ "o", "e", "na", "nas", "nos", "ao", "aos", "por", "foi", "era", "ser", "sou",
+ "tem", "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu",
+ "sua", "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais",
+ "já", "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
+ "olá"},
"it": {"il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
- "questo", "questa", "anche", "come", "più", "sono", "nella", "alla"},
+ "questo", "questa", "anche", "come", "più", "sono", "nella", "alla",
+ "la", "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi",
+ "ci", "ne", "ho", "hai", "ha", "abbiamo", "avete", "hanno", "era", "stato", "stata",
+ "devo", "deve", "devono", "voglio", "vorrei", "mio", "mia", "tuo", "sua", "quando",
+ "dove", "perche", "molto", "poco", "sempre", "mai", "già", "ancora", "adesso", "oggi",
+ "ieri", "grazie", "ciao", "scusa",
+ # the articulated prepositions: Italian-only words, which is what lets a state made of
+ # shared articles (`la fattura`) still name the language rather than stay undecided
+ "nel", "nell", "negli", "sul", "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei",
+ "delle", "dello", "degli", "agli", "alle", "col"},
"nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
"door", "maar", "ook", "worden", "deze", "naar", "wordt"},
# Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a
@@ -81,6 +118,13 @@ _NON_EN_DIACRITICS = set(
"āēģīķļņūž" # Baltic
"đ" # Serbo-Croatian / Vietnamese
)
+# Words that more than one list claims. `la`, `un`, `e`, `que`, `una` and friends are function words
+# of several of these languages at once, so matching one says "not English" without saying *which*
+# language: a shared word may not name a winner by itself (Romanian text was reported as French that
+# way), though it still counts toward the total of a language that also matched a word of its own.
+_SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}
+ if sum(w in words for words in _STOP.values()) > 1}
+
_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
@@ -162,6 +206,9 @@ def latin_profile(text: str) -> Dict[str, object]:
`looks_non_english`. `analyse` needs the evidence and not just the verdict, because
"undecided" and "English" are different answers and only one of them is safe to send to the
English checkpoint.
+
+ A non-English language is only named when it matched at least one word that no other list
+ claims: shared function words alone (`la`, `e`, `o`) identify no particular language.
"""
words = [w.lower() for w in _WORD.findall(text)]
lowered = text.lower()
@@ -174,13 +221,14 @@ def latin_profile(text: str) -> Dict[str, object]:
scores = {lg: sum(1 for w in words if w in sw) for lg, sw in _STOP.items()}
en = scores.get("en", 0)
- best_lg, best = max(((lg, s) for lg, s in scores.items() if lg != "en"),
- key=lambda kv: kv[1], default=(None, 0))
- # No stopword hit for any non-English language is no evidence for a *particular* one. Naming
- # the winner of a 0-0 tie invented a language (Romanian text was reported as French), so stay
- # undecided and let the diacritic rate speak.
- if best == 0:
- best_lg = None
+ # Only a language that matched at least one word no other list claims may be named. Without
+ # that condition the top score can be pure overlap -- `la` and `e` in Romanian text made
+ # Italian the winner -- which is a guess dressed as a detection. Such a language is dropped
+ # from the running rather than merely losing the tie, so a lesser score with real evidence
+ # still gets named, and the text stays undecided when no list has any.
+ evidenced = {lg: s for lg, s in scores.items()
+ if lg != "en" and any(w not in _SHARED_WORDS for w in set(words) & _STOP[lg])}
+ best_lg, best = max(evidenced.items(), key=lambda kv: kv[1], default=(None, 0))
lang = None
if best_lg and best >= max(2, en + 2):
@@ -199,8 +247,10 @@ def latin_profile(text: str) -> Dict[str, object]:
def guess_latin_language(text: str) -> Optional[str]:
"""Best-effort language code for Latin-script text, or None when undecided.
- Scores function-word hits per language and requires the winner to beat English by a margin,
- so ordinary English is never misrouted. Short inputs usually return None on purpose.
+ Scores function-word hits per language and requires the winner to beat English by a margin and
+ to have matched at least one word of its own, so ordinary English is never misrouted and a
+ word of several languages at once names none of them. Short inputs usually return None on
+ purpose.
"""
return latin_profile(text)["language"]
diff --git a/tests/test_router.py b/tests/test_router.py
index 0568f9ef..6ca0be7c 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -225,6 +225,67 @@ check("route/english still english",
check("route/short english still english", _r_lat.route("refund me").model, "english")
+# --------------------------------------------------------------------- plain-ASCII Romance (#172)
+# A state that lost its accents carries no diacritic rate for the non-English signal to read, and
+# mail clients and ticket systems strip them routinely, so the function-word lists are the only
+# evidence left. Spanish and Italian complaints were reported `is_english=True` and handed to the
+# English checkpoint -- the one that collapses off English -- while the accented spelling of the
+# same text reached multilingual. The lists for fr/es/pt/it held mostly accented words (`está`,
+# `más`, `è`, `être`, `não`) plus a handful of unaccented ones, so a stripped state matched one
+# word or none and fell under the two-hit margin below.
+for lang, text in [
+ ("es", "El pedido llego roto y nadie responde cuando escribo al soporte"),
+ ("es", "Quiero cancelar mi plan y pedir un reembolso"),
+ ("es", "La factura tiene un error en el importe total"),
+ ("es", "Necesito que me devuelvan el dinero de la compra duplicada"),
+ ("it", "Il cliente e stato addebitato due volte e vuole un rimborso"),
+ ("it", "Voglio cancellare il mio abbonamento e chiedere un rimborso"),
+ ("it", "La fattura contiene un errore nell importo totale"),
+ ("pt", "O cliente foi cobrado duas vezes e quer o dinheiro de volta"),
+ ("fr", "Le client a ete facture deux fois et demande un remboursement"),
+ ("fr", "Je ne peux pas acceder a mon compte et j ai besoin d aide"),
+]:
+ check("latin_lang/plain ascii " + lang + " " + text[:32], guess_latin_language(text), lang)
+ check("is_english/plain ascii " + lang + " " + text[:32], is_english(text), False)
+ check("route/plain ascii " + lang + " " + text[:32], _r_lat.route(text).model, "multilingual")
+# the accented spellings must keep working: those route on the diacritic rate
+for lang, text in [
+ ("es", "La facturación tiene un error y necesito una corrección urgente"),
+ ("it", "La fattura è sbagliata, devo avere un rimborso per il pagamento"),
+ ("fr", "La commande est arrivée cassée et personne ne répond au support"),
+]:
+ check("route/accented " + lang, _r_lat.route(text).model, "multilingual")
+
+# English must not move for this. The added words are ordinary English tokens as well -- `de facto`,
+# `et al.`, `e.g.`, `la carte`, `UN`, `MI5`, `DOS` -- and a state carrying none of them is the case
+# that has to keep routing to English.
+for text in [
+ "The customer was charged twice and wants a refund for this invoice",
+ "Please cancel my subscription and refund the duplicate charge today",
+ "The report by Smith et al. shows the de facto standard, e.g. the LA office and Rio",
+ "Our MI5 and UN contacts discussed the DOS attack in LA last month",
+ "No refund was issued, so I am writing to you again about invoice 4411",
+ "no refund no reply",
+ "The son of the director filed a complaint about the duplicate invoice",
+]:
+ check("is_english/romance control " + text[:32], is_english(text), True)
+ check("route/romance control " + text[:32], _r_lat.route(text).model, "english")
+
+# A word several lists claim (`la`, `e`, `o`) says "not English" without saying *which* language, so
+# it may not name one on its own -- the same rule as the 0-0 tie above, which is why the sample
+# below still names nothing even though `la` and `e` now count for Italian: it routes to multilingual
+# on the Romanian diacritic, not on a guessed language.
+check("latin_lang/shared words alone name nothing",
+ analyse("Cât e ora acum la Tokyo")["language"], None)
+check("route/shared words still multilingual",
+ _r_lat.route("Cât e ora acum la Tokyo").model, "multilingual")
+# ...but a distinctive word in the same state is enough to name the language it belongs to.
+check("latin_lang/distinctive word names the language",
+ guess_latin_language("La fattura contiene un errore nell importo totale"), "it")
+check("latin_lang/shared hits still count toward a named language",
+ guess_latin_language("La factura tiene un error en el importe total"), "es")
+
+
# --------------------------------------------------------------------- temperature clamp (#35)
# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which
# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.