Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): recognise Brazilian Portuguese support text (#202)

Paulo Carinhena33dcb14c6748
diff --git a/laya/lang.py b/laya/lang.py
index b01d1d4a..7edc3700 100644
--- a/laya/lang.py
+++ b/laya/lang.py
@@ -83,7 +83,19 @@ _STOP = {
"tem", "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu",
"sua", "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais",
"já", "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
- "olá"},
+ "olá",
+ # Brazilian support text: `você` and the unaccented `nao`/`voce`/`sao`/`ja` that a stripped
+ # state keeps (#172), and the chat abbreviations `vc`/`pra`. Without them "Voce pode me
+ # mandar a nota fiscal?" matched one word and went to the English checkpoint, which on
+ # `pt` reports 0.97 mean confidence at 0.47 accuracy. `ate`, `bom`, `sim` and `cade` stay
+ # out: each is an English token too (ate, BOM, SIM, Cade).
+ "você", "vocês", "voce", "voces", "vc", "vcs", "nao", "sao", "ja", "até", "tá", "pra",
+ "gostaria", "obrigada", "também", "tambem", "estou", "estamos", "meus", "minhas",
+ "nosso", "nossa", "consigo", "cadê", "boa", "tarde", "noite",
+ # the words a ticket keeps once the jargon is English ("Deu erro 500 no endpoint de login
+ # depois do update"): time and person words plus the past tenses a bug report is told in
+ "depois", "antes", "então", "entao", "ninguém", "ninguem", "alguém", "alguem", "nenhum",
+ "nenhuma", "estava", "ficou", "fiz", "deu"},
"it": {"il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
"questo", "questa", "anche", "come", "più", "sono", "nella", "alla",
"la", "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi",
diff --git a/tests/test_router.py b/tests/test_router.py
index ceb2acc6..114c49e0 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -316,6 +316,39 @@ check("latin_lang/shared hits still count toward a named language",
guess_latin_language("La factura tiene un error en el importe total"), "es")
+# --------------------------------------------------------------------- Brazilian support text
+# Short Brazilian messages lean on `você`/`vc`, the unaccented `nao`/`voce` and `gostaria`, none of
+# which the `pt` list held, so each matched one word, fell under the two-hit margin and went to the
+# English checkpoint -- which on `pt` reports 0.97 mean confidence at 0.47 accuracy (ECE 0.51).
+for text in [
+ "Boa tarde, gostaria de cancelar o plano",
+ "Voce pode me mandar a nota fiscal?",
+ "Você pode me mandar a nota fiscal?",
+ "Nao consigo fazer login no app",
+ "Pix nao caiu na conta",
+ "Gostaria de saber o prazo de entrega",
+ "Estou esperando faz uma semana",
+ "Vc pode cancelar pra mim?",
+ # a bug report whose jargon is English keeps only these words to say it is Portuguese
+ "Deu erro 500 no endpoint de login depois do update",
+ "Depois da atualizacao ninguem consegue logar",
+ "Antes funcionava, agora deu pau",
+ "Estava tudo certo ate a migracao",
+ "Entao o sistema travou de novo",
+]:
+ check("latin_lang/pt-br " + text[:32], guess_latin_language(text), "pt")
+ check("route/pt-br " + text[:32], _r_lat.route(text).model, "multilingual")
+# each added word that is also an English token must not move English text
+for text in [
+ "Our Sao Paulo office still has not received the invoice",
+ "My VC asked for the cap table and the invoice",
+ "Nossa Cafe charged my card twice this month",
+ "The Boa Vista branch reported an outage this morning",
+ "The pra team will review the claim tomorrow",
+]:
+ check("route/pt-br control " + text[:32], _r_lat.route(text).model, "english")
+
+
# --------------------------------------------------------------------- temperature clamp (#35)
# A fitted temperature below 1 sharpens logits. The shipped `choice:11+` bucket is 0.1006, which
# turned a 0.24 top probability into 0.99 confidence on 13-option skill routing.