Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): a dotted token is an identifier, not a prose word (#179)

devloper961-makerec9cb67587a6
diff --git a/laya/lang.py b/laya/lang.py
index a030fafc..b01d1d4a 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -126,6 +126,13 @@ _SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}
if sum(w in words for words in _STOP.values()) > 1}
_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
+# A token whose dot or @ joins word characters is an identifier, not prose: `github.com`,
+# `[email protected]`, `v1.2.3`, `U.S.A.`. `_WORD` splits them into pieces that collide with real
+# function words -- `com` is Portuguese for "with", `o` is its article, `e` is Italian "e" -- so a
+# state that was mostly links scored a language it does not contain, and two domains were enough to
+# cross the margin below. A sentence-final period (`arrivato.`) keeps its word: the pattern needs
+# word characters on both sides of the dot.
+_IDENTIFIER = re.compile(r"[\w-]*(?:[.@][\w-]+)+", re.UNICODE)
def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str]:
@@ -210,7 +217,7 @@ def latin_profile(text: str) -> Dict[str, object]:
A non-English language is only named when it matched at least one word that no other list
claims: shared function words alone (`la`, `e`, `o`) identify no particular language.
"""
- words = [w.lower() for w in _WORD.findall(text)]
+ words = [w.lower() for w in _WORD.findall(_IDENTIFIER.sub(" ", text))]
lowered = text.lower()
diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)
diac_rate = diac / max(1, len(lowered))
diff --git a/tests/test_router.py b/tests/test_router.py
index 9e9ffff6..ceb2acc6 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -116,6 +116,36 @@ check("latin_lang/long english stays en",
"we have been waiting for three days and nobody has replied to us"), "en")
+# --------------------------------------------------------------------- dotted tokens (#177)
+# `com` is Portuguese ("with") and `o` is its article, and `_WORD` splits `github.com` into
+# `github` + `com`, so every domain in a state scored a Portuguese hit: two of them crossed the
+# margin and sent an English state with links in it to the multilingual checkpoint, reported as
+# Portuguese. A token whose dot or @ joins word characters is an identifier, not prose.
+_r_dotted = Router()
+for label, state in [
+ ("url and email fields", {"url": "github.com", "email": "[email protected]"}),
+ ("two bare domains", "github.com acme.com"),
+ ("link list", {"links": ["example.com", "example.co.uk", "docs.readthedocs.io"]}),
+]:
+ check("latin_lang/dotted " + label, analyse(state)["language"], None)
+ check("is_english/dotted " + label, is_english(state), True)
+ check("route/dotted " + label, _r_dotted.route(state).model, "english")
+# versions, decimals and dotted abbreviations are identifiers too, and were never prose
+for text, label in [("build 1.2.3 on 12.30 with ratio 0.5", "version and decimal"),
+ ("Report by Smith et al., e.g. the U.S.A. office", "dotted abbreviation")]:
+ check("is_english/dotted " + label, is_english(text), True)
+ check("route/dotted " + label, _r_dotted.route(text).model, "english")
+# masking identifiers must not cost the prose around them its language, and a full stop ends a
+# sentence rather than joining an identifier: the word before it keeps its letters.
+check("latin_lang/portuguese prose with a link",
+ guess_latin_language("O cliente nao recebeu o produto, mas quer o dinheiro para a conta, "
+ "veja example.com"), "pt")
+check("latin_lang/portuguese sentence with a full stop",
+ guess_latin_language("O cliente nao recebeu o produto, mas quer o dinheiro para a conta."), "pt")
+check("route/english with a link stays english",
+ _r_dotted.route({"body": "Please check example.com and acme.com for the invoice"}).model, "english")
+
+
# --------------------------------------------------------------------- state flattening
check("state_text/dict", "charged twice" in state_text({"body": "charged twice", "n": 3}), True)
check("state_text/nested", "deep" in state_text({"a": {"b": ["deep"]}}), True)