Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): a dotted token is an identifier, not a prose word (#179)
devloper961-makerec9cb67587a6
diff --git a/laya/lang.py b/laya/lang.pyindex a030fafc..b01d1d4a 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -126,6 +126,13 @@ _SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}if sum(w in words for words in _STOP.values()) > 1}_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)+# A token whose dot or @ joins word characters is an identifier, not prose: `github.com`,+# `[email protected]`, `v1.2.3`, `U.S.A.`. `_WORD` splits them into pieces that collide with real+# function words -- `com` is Portuguese for "with", `o` is its article, `e` is Italian "e" -- so a+# state that was mostly links scored a language it does not contain, and two domains were enough to+# cross the margin below. A sentence-final period (`arrivato.`) keeps its word: the pattern needs+# word characters on both sides of the dot.+_IDENTIFIER = re.compile(r"[\w-]*(?:[.@][\w-]+)+", re.UNICODE)def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str]:@@ -210,7 +217,7 @@ def latin_profile(text: str) -> Dict[str, object]:A non-English language is only named when it matched at least one word that no other listclaims: shared function words alone (`la`, `e`, `o`) identify no particular language."""- words = [w.lower() for w in _WORD.findall(text)]+ words = [w.lower() for w in _WORD.findall(_IDENTIFIER.sub(" ", text))]lowered = text.lower()diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)diac_rate = diac / max(1, len(lowered))diff --git a/tests/test_router.py b/tests/test_router.pyindex 9e9ffff6..ceb2acc6 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -116,6 +116,36 @@ check("latin_lang/long english stays en","we have been waiting for three days and nobody has replied to us"), "en")+# --------------------------------------------------------------------- dotted tokens (#177)+# `com` is Portuguese ("with") and `o` is its article, and `_WORD` splits `github.com` into+# `github` + `com`, so every domain in a state scored a Portuguese hit: two of them crossed the+# margin and sent an English state with links in it to the multilingual checkpoint, reported as+# Portuguese. A token whose dot or @ joins word characters is an identifier, not prose.+_r_dotted = Router()+for label, state in [+ ("url and email fields", {"url": "github.com", "email": "[email protected]"}),+ ("two bare domains", "github.com acme.com"),+ ("link list", {"links": ["example.com", "example.co.uk", "docs.readthedocs.io"]}),+]:+ check("latin_lang/dotted " + label, analyse(state)["language"], None)+ check("is_english/dotted " + label, is_english(state), True)+ check("route/dotted " + label, _r_dotted.route(state).model, "english")+# versions, decimals and dotted abbreviations are identifiers too, and were never prose+for text, label in [("build 1.2.3 on 12.30 with ratio 0.5", "version and decimal"),+ ("Report by Smith et al., e.g. the U.S.A. office", "dotted abbreviation")]:+ check("is_english/dotted " + label, is_english(text), True)+ check("route/dotted " + label, _r_dotted.route(text).model, "english")+# masking identifiers must not cost the prose around them its language, and a full stop ends a+# sentence rather than joining an identifier: the word before it keeps its letters.+check("latin_lang/portuguese prose with a link",+ guess_latin_language("O cliente nao recebeu o produto, mas quer o dinheiro para a conta, "+ "veja example.com"), "pt")+check("latin_lang/portuguese sentence with a full stop",+ guess_latin_language("O cliente nao recebeu o produto, mas quer o dinheiro para a conta."), "pt")+check("route/english with a link stays english",+ _r_dotted.route({"body": "Please check example.com and acme.com for the invoice"}).model, "english")++# --------------------------------------------------------------------- state flatteningcheck("state_text/dict", "charged twice" in state_text({"body": "charged twice", "n": 3}), True)check("state_text/nested", "deep" in state_text({"a": {"b": ["deep"]}}), True)