Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): recognise Brazilian Portuguese support text (#202)
Paulo Carinhena33dcb14c6748
diff --git a/laya/lang.py b/laya/lang.pyindex b01d1d4a..7edc3700 100644--- a/laya/lang.py+++ b/laya/lang.py@@ -83,7 +83,19 @@ _STOP = {"tem", "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu","sua", "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais","já", "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",- "olá"},+ "olá",+ # Brazilian support text: `você` and the unaccented `nao`/`voce`/`sao`/`ja` that a stripped+ # state keeps (#172), and the chat abbreviations `vc`/`pra`. Without them "Voce pode me+ # mandar a nota fiscal?" matched one word and went to the English checkpoint, which on+ # `pt` reports 0.97 mean confidence at 0.47 accuracy. `ate`, `bom`, `sim` and `cade` stay+ # out: each is an English token too (ate, BOM, SIM, Cade).+ "você", "vocês", "voce", "voces", "vc", "vcs", "nao", "sao", "ja", "até", "tá", "pra",+ "gostaria", "obrigada", "também", "tambem", "estou", "estamos", "meus", "minhas",+ "nosso", "nossa", "consigo", "cadê", "boa", "tarde", "noite",+ # the words a ticket keeps once the jargon is English ("Deu erro 500 no endpoint de login+ # depois do update"): time and person words plus the past tenses a bug report is told in+ "depois", "antes", "então", "entao", "ninguém", "ninguem", "alguém", "alguem", "nenhum",+ "nenhuma", "estava", "ficou", "fiz", "deu"},"it": {"il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono","questo", "questa", "anche", "come", "più", "sono", "nella", "alla","la", "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi",diff --git a/tests/test_router.py b/tests/test_router.pyindex ceb2acc6..114c49e0 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -316,6 +316,39 @@ check("latin_lang/shared hits still count toward a named language",guess_latin_language("La factura tiene un error en el importe total"), "es")+# --------------------------------------------------------------------- Brazilian support text+# Short Brazilian messages lean on `você`/`vc`, the unaccented `nao`/`voce` and `gostaria`, none of+# which the `pt` list held, so each matched one word, fell under the two-hit margin and went to the+# English checkpoint -- which on `pt` reports 0.97 mean confidence at 0.47 accuracy (ECE 0.51).+for text in [+ "Boa tarde, gostaria de cancelar o plano",+ "Voce pode me mandar a nota fiscal?",+ "Você pode me mandar a nota fiscal?",+ "Nao consigo fazer login no app",+ "Pix nao caiu na conta",+ "Gostaria de saber o prazo de entrega",+ "Estou esperando faz uma semana",+ "Vc pode cancelar pra mim?",+ # a bug report whose jargon is English keeps only these words to say it is Portuguese+ "Deu erro 500 no endpoint de login depois do update",+ "Depois da atualizacao ninguem consegue logar",+ "Antes funcionava, agora deu pau",+ "Estava tudo certo ate a migracao",+ "Entao o sistema travou de novo",+]:+ check("latin_lang/pt-br " + text[:32], guess_latin_language(text), "pt")+ check("route/pt-br " + text[:32], _r_lat.route(text).model, "multilingual")+# each added word that is also an English token must not move English text+for text in [+ "Our Sao Paulo office still has not received the invoice",+ "My VC asked for the cap table and the invoice",+ "Nossa Cafe charged my card twice this month",+ "The Boa Vista branch reported an outage this morning",+ "The pra team will review the claim tomorrow",+]:+ check("route/pt-br control " + text[:32], _r_lat.route(text).model, "english")++# --------------------------------------------------------------------- temperature clamp (#35)# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.