Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): a repeated English word that collides with a foreign function word routed English to multilingual

AlKor1312be45494ca8
diff --git a/laya/lang.py b/laya/lang.py
index d457940b..8522bf93 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -12,6 +12,7 @@ is explicitly best-effort: pass an explicit model or `lang=` when you already kn
"""
import re
import unicodedata
+from collections import Counter
from collections.abc import Mapping
from typing import Dict, List, Optional, Union
@@ -176,6 +177,18 @@ _SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}
# English function words no other list holds (`in`, `is`, `as`, `was` are shared with German,
# Dutch and Portuguese). They alone carry the English rescue of `latin_profile`.
_EN_ONLY_WORDS = _STOP["en"] - _SHARED_WORDS
+# A few of these function words are also ordinary English words: `come` (Italian), `son` (Spanish),
+# `do` (Portuguese), `care` (Romanian), `todo` (a to-do list), `per`, `plus`, and `im` -- German
+# `im`, and also how `I'm` is written without the apostrophe. For those a repeat is evidence of the
+# foreign language exactly once: "do more, do less" repeats an English word, and counting both hits
+# let one such word clear the `best >= 2` bar below and send plain English to the multilingual
+# checkpoint. Every other word keeps counting occurrences: `der`, `los` and `sa` are nobody's
+# English, so a request whose only evidence is `der` twice ("reduzieren der helligkeit der lichter")
+# is still German, and deduping every word instead sent 791 of 148,700 non-English MASSIVE test rows
+# to the English checkpoint. Dutch `van` is the word this list deliberately leaves out: "liedje van
+# ... van" is ordinary Dutch, and deduping it moved 14 of those rows while rescuing no English one.
+_EN_COLLISION_WORDS = {"come", "son", "do", "care", "todo", "im", "per", "plus"} & {
+ w for lg, sw in _STOP.items() if lg != "en" for w in sw}
_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
# A token whose dot or @ joins word characters is an identifier, not prose: `github.com`,
@@ -400,7 +413,10 @@ def latin_profile(text: str) -> Dict[str, object]:
return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,
"looks_non_english": non_english}
- scores = {lg: sum(1 for w in words if w in sw) for lg, sw in _STOP.items()}
+ # A collision word counts once however often it repeats; every other word counts its hits.
+ counts = Counter(words)
+ scores = {lg: sum(1 if w in _EN_COLLISION_WORDS else n for w, n in counts.items() if w in sw)
+ for lg, sw in _STOP.items()}
en = scores.get("en", 0)
# Only a language that matched at least one word no other list claims may be named. Without
# that condition the top score can be pure overlap -- `la` and `e` in Romanian text made
diff --git a/tests/test_lang_guess.py b/tests/test_lang_guess.py
index 17c20d69..d68a3d83 100644
--- a/tests/test_lang_guess.py
+++ b/tests/test_lang_guess.py
@@ -18,6 +18,7 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from laya.router import Router, _english_from_code # noqa: E402
+from laya.lang import analyse # noqa: E402
PASS, FAIL = [], []
@@ -176,6 +177,51 @@ for label, s, want in BEFORE:
check("unchanged/" + label, r0.route(s, GENERIC)["model"], want)
+# A foreign function word that is also an ordinary English word (`come` it, `son` es, `do` pt,
+# `care` ro, `todo`/`im`/`per`/`plus`) used to name that language when it merely appeared
+# twice, because the score counted occurrences and `best >= 2` was reached by one repeated word.
+# Ordinary English then routed to the multilingual checkpoint. Such a word now counts once however
+# often it repeats, so one collision no longer clears the bar.
+ENGLISH_ON_A_REPEAT = [
+ "Come one, come all", # come (it) x2
+ "My son, your son", # son (es) x2
+ "Do more, do less", # do (pt) x2
+ "Care more, care less", # care (ro) x2
+ "Add a todo, then another todo item", # todo (es) x2
+ "im not able to log in, im stuck", # im (de) x2
+ "add 45 to 87 plus 54 plus 43 plus 22", # plus (de, fr) x3 -- a CLINC150 test row
+]
+for s in ENGLISH_ON_A_REPEAT:
+ check("repeat/plain english stays english: %r" % s, analyse(s)["is_english"], True)
+
+# The dedupe is limited to those words. A function word that is nobody's English -- `der`, `des`,
+# `di`, `sa` -- still counts every occurrence, so a real request whose only evidence is one such
+# word repeated stays foreign. Rows from the MASSIVE test splits, which is where deduping every
+# word instead of the collisions cost 791 of 148,700 non-English texts their checkpoint.
+REPEAT_STAYS_FOREIGN = [
+ ("de", "reduzieren der helligkeit der lichter"),
+ ("es", "enumerar todos los horarios de los tren a nueva york"),
+ ("fr", "jouer des chansons des beatles"),
+ ("it", "numero di telefono di giacomo"),
+ ("pt", "pede um pacote de massa chinesa faz um pedido takeaway"),
+ ("ro", "ar trebui sa port o pelerina de ploaie inainte sa ies afara"),
+ ("nl", "herinner me eraan dat ik dat liedje leuk vind"),
+ # `van` is left out of the collision list for exactly this row: it is ordinary Dutch
+ ("nl", "hey olly ik hou van muziek van frans bauer"),
+]
+for want, s in REPEAT_STAYS_FOREIGN:
+ check("repeat/%s on a repeated word is still foreign" % want, analyse(s)["is_english"], False)
+
+# and real foreign requests -- two different function words each -- are still detected
+STILL_FOREIGN = [
+ ("es", "Hola, necesito cancelar mi pedido por favor ahora mismo gracias"),
+ ("pt", "Eu quero cancelar o meu pedido por favor agora mesmo obrigado"),
+ ("it", "Vorrei annullare il mio ordine per favore adesso grazie mille"),
+]
+for want, s in STILL_FOREIGN:
+ check("repeat/genuine %s still foreign" % want, analyse(s)["is_english"], False)
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)