Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(ts): port mixed-segment detection from Python analyse() (#384)
Swatantra Yadave37428a124f9
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 94fec7b7..6ab5a0af 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -325,6 +325,70 @@ export interface AnalyseResult {languageUndecided: boolean;diacriticRate: number;nonLatinFraction: number;+ mixedSegment: string | null;+}++// Code is not prose in any language, but split into words it reads as one: `os.path` is Portuguese+// (`os`), `round(el, 2)` Spanish (`el`), `non_english` Italian (`non`). A line pasted from a program+// into an English request must not count as a foreign segment, so a line carrying code syntax --+// `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers+// are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.+const CODE_LINE_RE = /[=;{}[\]]|\w\(/;+// Slash and backslash compounds are names, not sentences: `Nav/Com` and `OS/2` read as Portuguese+// (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`). A whitespace token+// holding a letter or digit, a joiner (`.`, `_`, `/`, `\`) and another letter or digit is an+// identifier or a compound and is dropped whole.+const JOINED_RE = /[^\W_][._/\\][^\W_]/;+// An all-caps token inside mixed-case text is an acronym or a code: `MON`, `LA`, `EST`, `COM`, `DES`+// are hockey teams, states, time zones and radio bands, not French or Portuguese. A segment written+// entirely in capitals keeps its words -- a customer shouting in Portuguese is still Portuguese.+const LETTER_RUN_RE = /[\p{L}]{2,}/gu;++/** Language code for one non-code line, or null when it does not name a foreign language.+ *+ * Same evidence bar as `nonEnglishSegment`: four words, a language `latinProfile` will name,+ * and two *different* words of that language. Acronyms and slash compounds are not words.+ */+function namedProseLanguage(segment: string): string | null {+ if (!segment.trim() || CODE_LINE_RE.test(segment)) return null;+ const prose = segment.split(/\s+/).filter((tok) => !JOINED_RE.test(tok)).join(" ");+ // In mixed-case text, replace all-caps runs with spaces (they are acronyms).+ let cleaned = prose;+ if ([...prose].some((ch) => /\p{Ll}/u.test(ch))) {+ cleaned = prose.replace(LETTER_RUN_RE, (m) => (m === m.toUpperCase() ? " " : m));+ }+ const tokens = cleaned.match(WORD_RE) ?? [];+ if (tokens.length < 4) return null;+ const lang = latinProfile(cleaned).language;+ if (lang === null || lang === "en") return null;+ const stopSet = STOP[lang];+ if (!stopSet) return null;+ const distinctHits = new Set(tokens.map((w) => w.toLowerCase()).filter((w) => stopSet.has(w)));+ if (distinctHits.size < 2) return null;+ return lang;+}++/** First line or field that, read on its own, is named a non-English language, else null.+ *+ * Returns [language, segment]. A segment needs the evidence a whole state needs -- at least four+ * words, and a language named by `latinProfile` -- and, because one line carries far less text+ * than a state, two things more: the words that name the language must be two *different* ones+ * (`COM ... COM` in an English radio listing is one word seen twice), and acronyms and slash+ * compounds are not words. This adds no new way to call English text foreign; it only stops a+ * longer English part from outvoting a foreign one. Reads at most `maxChars` characters in all.+ */+function nonEnglishSegment(state: unknown, maxChars = 4000): [string, string] | null {+ let seen = 0;+ for (const leaf of iterText(state)) {+ for (const seg of leaf.split("\n")) {+ if (seen >= maxChars) return null;+ const capped = seg.slice(0, maxChars - seen);+ seen += capped.length;+ const lang = namedProseLanguage(capped);+ if (lang) return [lang, capped.trim()];+ }+ }+ return null;}function round4(x: number): number {@@ -357,14 +421,14 @@ function analyseText(text: string): AnalyseResult {return {script: "unknown", scriptProfile: prof, language: null,isEnglish: true, languageUndecided: true, diacriticRate: 0.0,- nonLatinFraction: 0.0,+ nonLatinFraction: 0.0, mixedSegment: null,};}if (script !== "latin") {return {script, scriptProfile: prof, language: null,isEnglish: false, languageUndecided: true, diacriticRate: 0.0,- nonLatinFraction: nonLatin,+ nonLatinFraction: nonLatin, mixedSegment: null,};}const profLat = latinProfile(text);@@ -375,7 +439,7 @@ function analyseText(text: string): AnalyseResult {script: "latin", scriptProfile: prof, language: lang,isEnglish: english, languageUndecided: undecided,diacriticRate: round4(profLat.diacriticRate),- nonLatinFraction: nonLatin,+ nonLatinFraction: nonLatin, mixedSegment: null,};}@@ -409,6 +473,29 @@ export function analyse(state: unknown): AnalyseResult {// note fill the window, or outvote a short German message, and that message was// then sent to the English checkpoint.const result = analyseText(stateText(state));+ if (result.script === "latin" && result.isEnglish) {+ // A Portuguese ticket with an English stack trace, error payload or form template reads as+ // English as a whole, because the English part is longer -- yet the part a question is about+ // is the customer's, and the English checkpoint cannot read it. So a state that would go to+ // English is checked line by line and field by field.+ const leaves = iterText(state);+ // a single line has no other part to be outvoted by, and was just read whole+ if (leaves.length > 1 || leaves.some((leaf) => leaf.includes("\n"))) {+ const found = nonEnglishSegment(state);+ if (found) {+ const [lang, mixed] = found;+ return {+ ...result,+ language: lang,+ isEnglish: false,+ languageUndecided: false,+ mixedSegment: mixed,+ };+ }+ }+ }+ // A plain string was just read whole. A structured state can still hide a message past the+ // segment cap, or in a script `latinProfile` does not name.if (typeof state === "string" || state == null || state instanceof Uint8Array || !result.isEnglish) {return result;}diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.tsindex 0f09b078..09bf951d 100644--- a/laya-ts/src/router.ts+++ b/laya-ts/src/router.ts@@ -444,7 +444,11 @@ export class Router extends HookRegistry {"the English checkpoint cannot read it";} else if (!det.isEnglish) {key = "multilingual";- if (det.language) {+ if (det.mixedSegment) {+ reason =+ `Latin script, mostly English, but a line or field reads as ${JSON.stringify(det.language)} ` ++ `(${JSON.stringify(det.mixedSegment.slice(0, 60))}); the English checkpoint cannot read it`;+ } else if (det.language) {reason = `Latin script but language looks like ${JSON.stringify(det.language)}, not English`;} else {reason =diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex 342eabab..3ba897b6 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -110,4 +110,86 @@ describe("lang", () => {expect(buried.isEnglish).toBe(false);expect(buried.language).toBe("de");});++ // mixed-segment detection (port of Python's _non_english_segment)+ // A foreign line embedded in an English ticket must flip isEnglish and record the segment.+ const TRACE = "I checked the refund status and here is what I found for the customer.\n" ++ "The charge cleared last Tuesday and the bank confirmed it went through.\n" ++ "Quero o meu dinheiro de volta agora mesmo porque já esperei demais\n" ++ "Please escalate to the billing manager if the refund has not been issued by Friday.";++ it("mixed-segment: Portuguese line in English ticket is not english", () => {+ const a = analyse(TRACE);+ expect(a.isEnglish).toBe(false);+ expect(a.language).toBe("pt");+ expect(a.mixedSegment).toBe(+ "Quero o meu dinheiro de volta agora mesmo porque já esperei demais",+ );+ });++ it("mixed-segment: German error payload in English ticket", () => {+ const state = "The integration test failed with this error on the staging server " ++ "and I am not sure whether it is a data issue or a code regression.\n" ++ "Fehler: Die Verbindung zum Server wurde unterbrochen, bitte versuchen Sie es spaeter noch einmal";+ const a = analyse(state);+ expect(a.isEnglish).toBe(false);+ expect(a.mixedSegment).toContain("Verbindung");+ });++ it("mixed-segment: Spanish error in structured state", () => {+ const state = {+ subject: "Payment failed for a customer in Madrid",+ description: "The customer tried three times with the same card and each attempt was declined by " ++ "the gateway, so we would like to know whether the problem is on our side or with the bank.",+ error: {+ code: "card_declined",+ message: "La tarjeta fue rechazada por el banco emisor, contacte con su banco",+ },+ };+ const a = analyse(state);+ expect(a.isEnglish).toBe(false);+ expect(a.mixedSegment).toContain("tarjeta");+ });++ it("mixed-segment: all-caps Portuguese line", () => {+ const state =+ "This is the fourth email I have sent about the same order and nobody has answered any of them.\n" ++ "The customer wrote this in the chat and then closed the window:\n" ++ "QUERO MEU DINHEIRO DE VOLTA AGORA\n" ++ "Could someone from the billing team look at order 5512 today?";+ const a = analyse(state);+ expect(a.isEnglish).toBe(false);+ expect(a.mixedSegment).toBe("QUERO MEU DINHEIRO DE VOLTA AGORA");+ });++ // English stays English: multi-line, short foreign sign-off, code lines+ it.each([+ ["multi-line english", "Hi team,\nThe export failed again last night.\nCan you check the logs?\nThanks"],+ ["short portuguese sign-off", "Please resend the invoice for March, the amount is wrong.\nAtenciosamente, Joao"],+ ["os.path", "The build broke after the refactor.\nREPO = os.path.dirname(os.path.dirname(__file__))\n" ++ "Please take a look at the import paths when you can."],+ ["round(el)", "The latency script crashes on large runs.\nmix[key] = {\"total_s\": round(el, 2)}\n" ++ "Can you check why the stream is empty?"],+ ["same word twice (Nav/Com)", "I'm looking for good deals on the following (used or new):\n" ++ "Aviation Headsets (with mic).\n" ++ "Handheld Nav/Com tranciever (may consider COM only).\nPortable GPS or Loran Navigator."],+ ["team codes", "Round two predictions for the pool, as promised.\nQUE vs MON: MON in 7.\n" ++ "PIT vs NYI: PIT in 5."],+ ["slash compound", "I need a converter for these image formats.\n" ++ "DOS, OS/2 or platform independent programs if possible.\nThanks in advance."],+ ["backslash path", "My modem stopped answering after the upgrade.\nC:\\DOS\\mode COM1:9600,n,8,1,p\n" ++ "Is that the right line for a 9600 baud connection?"],+ ])("mixed-segment english stays english: %s", (_label, state) => {+ expect(analyse(state).isEnglish).toBe(true);+ expect(analyse(state).mixedSegment).toBeNull();+ });++ // The segment check reads at most 4000 chars, so a foreign line past that cap is not seen.+ it("mixed-segment: segment check reads at most the cap", () => {+ const state = {+ log: "The export failed again last night for the whole region. ".repeat(80),+ body: "Quero cancelar meu plano agora mesmo",+ };+ expect(analyse(state).mixedSegment).toBeNull();+ });});