Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): name plain-ASCII German the way Python does (#130 follow-up)
李氵氵77a4cf8cdd95
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 03d38ea9..4a5e50c6 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -40,7 +40,13 @@ const STOP: Record<string, Set<string>> = {"merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi","alors", "donc"]),de: new Set(["der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",- "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber"]),+ "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",+ "ich", "wir", "mir", "mich", "dir", "dich", "uns", "mein", "meine", "meinen",+ "meinem", "meiner", "diese", "dieser", "diesen", "dieses", "einen", "einem", "einer",+ "wie", "wo", "wann", "welche", "im", "zum", "zur", "aus", "bei", "nach", "noch", "bitte",+ "heute", "jetzt", "kann", "kannst", "habe", "gibt", "wird",+ // shared with English on purpose: counted for English alone, they outvoted short German+ "in", "was"]),es: new Set(["el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como","pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus","la", "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos",diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex 9cfce581..120018fe 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -1,6 +1,6 @@// laya-ts/tests/lang.test.tsimport { describe, expect, it } from "vitest";-import { analyse, detectScript, isEnglish } from "../src/lang.js";+import { analyse, detectScript, guessLatinLanguage, isEnglish } from "../src/lang.js";describe("lang", () => {it("detects devanagari as non-latin", () => {expect(detectScript("मुझसे दो बार शुल्क लिया गया")).toBe("devanagari");@@ -39,4 +39,34 @@ describe("lang", () => {it("counts IPA extensions as latin, like Python", () => {expect(detectScript("ɑ ɒ ɛ ɔ ɪ ʊ æ ʃ θ ð")).toBe("latin");});+ // Plain-ASCII German: no umlaut, so the diacritic rate is 0 and the function words are the only+ // evidence. Python names these `de` (laya/lang.py, #130); with the 23-word list they read as+ // English here and the Router sent them to the English checkpoint.+ it("names German that carries no diacritics", () => {+ for (const text of [+ "wie lautet die temperatur in fulda in hessen",+ "Mein Konto wurde zweimal belastet, bitte erstatten Sie",+ "Ich brauche eine Rechnung fuer meine letzte Bestellung",+ "Kann ich meine Bestellung noch heute stornieren",+ "Bitte senden Sie mir eine neue Kreditkartenabrechnung",+ ]) {+ const a = analyse(text);+ expect(a.language, text).toBe("de");+ expect(a.isEnglish, text).toBe(false);+ expect(guessLatinLanguage(text), text).toBe("de");+ }+ });+ // `in` and `was` count for German as well as English, and `im` and `den` are English tokens too.+ // Pin the English states that flip if any of them becomes German-only.+ it("keeps English that shares words with the German list", () => {+ expect(analyse("turn off smart lamp in den").isEnglish).toBe(true);+ expect(analyse("What was the reason for the delay").language).toBe("en");+ expect(analyse("Where can I find my invoice in the app").isEnglish).toBe(true);+ });+ // `es` and `du` are German function words, but Spanish and French claim them, and a word two+ // lists share names neither language. They stay out of the German list.+ it("does not pull es or du into German", () => {+ expect(analyse("que hora es en australia").language).toBe("es");+ expect(analyse("baisse le volume du haut-parleur").language).toBe("fr");+ });});