Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): C/POSIX/und/zxx/mul lang codes abstain like Python (follow-up to #368)
thejenilsoni4db74232e336
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.tsindex 64c6552d..c20cdc7f 100644--- a/laya-ts/src/router.ts+++ b/laya-ts/src/router.ts@@ -77,13 +77,19 @@ export function matchTypedDecisionsWorkflow(const ENGLISH_SUBTAGS = new Set(["en", "eng", "english"]);+// Valid `$LANG` values that name no language, so they answer nothing about the state: `C`,+// `POSIX` and `C.UTF-8` (the official Python image's default), plus the ISO 639-2 special codes+// `und` (undetermined), `zxx` (no linguistic content) and `mul` (multiple). They abstain like a+// blank code instead of forcing the multilingual checkpoint on English text (Python parity).+const LANGUAGE_AGNOSTIC_CODES = new Set(["c", "posix", "und", "zxx", "mul"]);+export function englishFromCode(value: unknown): boolean | null {if (value === null || value === undefined) return null;let code = String(value).trim().toLowerCase();if (!code) return null;code = code.split(".", 1)[0]; // en_US.UTF-8 -> en_USconst primary = code.replace(/_/g, "-").split("-", 1)[0]; // en_US -> en- if (!primary) return null;+ if (!primary || LANGUAGE_AGNOSTIC_CODES.has(primary)) return null;return ENGLISH_SUBTAGS.has(primary);}diff --git a/laya-ts/tests/lang-agnostic-codes.test.ts b/laya-ts/tests/lang-agnostic-codes.test.tsnew file mode 100644index 00000000..28d187e1--- /dev/null+++ b/laya-ts/tests/lang-agnostic-codes.test.ts@@ -0,0 +1,54 @@+import { describe, expect, it } from "vitest";+import { Router, englishFromCode } from "../src/router.js";++// Parity with the Python fix for #359: `C`, `POSIX` and `C.UTF-8` are valid `$LANG` values that+// name no language (`C.UTF-8` is the official Python image's default), and the ISO 639-2 special+// codes `und`/`zxx`/`mul` say the same. They abstain, so detection names the checkpoint instead+// of every request being pinned to the multilingual one.+const ENGLISH_STATE = "Please refund the duplicate charge on invoice 4411";++describe("lang codes that name no language", () => {+ const agnostic = ["C", "POSIX", "C.UTF-8", "c.utf8", "c", "posix", "und", "zxx", "mul", "UND", "Zxx", " und "];++ for (const code of agnostic) {+ it(`${JSON.stringify(code)} abstains and lets detection decide`, () => {+ expect(englishFromCode(code)).toBeNull();+ for (const opts of [{ lang: code }, { langGuess: code }]) {+ const d = new Router().route(ENGLISH_STATE, {}, opts);+ expect(d.model).toBe("english");+ expect(d.reason).not.toContain("explicit");+ expect(d.detection).not.toBeNull();+ }+ const installed = new Router({ langGuess: code }).route(ENGLISH_STATE, {});+ expect(installed.model).toBe("english");+ expect(installed.detection).not.toBeNull();+ });+ }++ it("blank codes still abstain", () => {+ expect(englishFromCode("")).toBeNull();+ expect(englishFromCode(null)).toBeNull();+ expect(englishFromCode(" ")).toBeNull();+ });++ for (const code of ["en", "eng", "english", "EN", "en-US", "en_US.UTF-8"]) {+ it(`${code} is still decisive English`, () => {+ expect(englishFromCode(code)).toBe(true);+ expect(new Router().route(ENGLISH_STATE, {}, { lang: code }).reason).toContain("explicit");+ });+ }++ for (const code of ["de", "fr", "zh", "ja", "pt-BR", "de_DE.UTF-8"]) {+ it(`${code} is still decisive non-English`, () => {+ expect(englishFromCode(code)).toBe(false);+ expect(new Router().route(ENGLISH_STATE, {}, { lang: code }).model).toBe("multilingual");+ });+ }++ it("the primary subtag is what is compared", () => {+ expect(englishFromCode("POSIX-1")).toBeNull();+ // `ca` (Catalan) and `mn` (Mongolian) share letters with agnostic codes but name languages.+ expect(englishFromCode("ca")).toBe(false);+ expect(englishFromCode("mn")).toBe(false);+ });+});