Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): C/POSIX/und/zxx/mul lang codes abstain like Python (follow-up to #368)

thejenilsoni4db74232e336
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts
index 64c6552d..c20cdc7f 100644
--- a/laya-ts/src/router.ts
+++ b/laya-ts/src/router.ts
@@ -77,13 +77,19 @@ export function matchTypedDecisionsWorkflow(
const ENGLISH_SUBTAGS = new Set(["en", "eng", "english"]);
+// Valid `$LANG` values that name no language, so they answer nothing about the state: `C`,
+// `POSIX` and `C.UTF-8` (the official Python image's default), plus the ISO 639-2 special codes
+// `und` (undetermined), `zxx` (no linguistic content) and `mul` (multiple). They abstain like a
+// blank code instead of forcing the multilingual checkpoint on English text (Python parity).
+const LANGUAGE_AGNOSTIC_CODES = new Set(["c", "posix", "und", "zxx", "mul"]);
+
export function englishFromCode(value: unknown): boolean | null {
if (value === null || value === undefined) return null;
let code = String(value).trim().toLowerCase();
if (!code) return null;
code = code.split(".", 1)[0]; // en_US.UTF-8 -> en_US
const primary = code.replace(/_/g, "-").split("-", 1)[0]; // en_US -> en
- if (!primary) return null;
+ if (!primary || LANGUAGE_AGNOSTIC_CODES.has(primary)) return null;
return ENGLISH_SUBTAGS.has(primary);
}
diff --git a/laya-ts/tests/lang-agnostic-codes.test.ts b/laya-ts/tests/lang-agnostic-codes.test.ts
new file mode 100644
index 00000000..28d187e1
--- /dev/null
+++ b/laya-ts/tests/lang-agnostic-codes.test.ts
@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+import { Router, englishFromCode } from "../src/router.js";
+
+// Parity with the Python fix for #359: `C`, `POSIX` and `C.UTF-8` are valid `$LANG` values that
+// name no language (`C.UTF-8` is the official Python image's default), and the ISO 639-2 special
+// codes `und`/`zxx`/`mul` say the same. They abstain, so detection names the checkpoint instead
+// of every request being pinned to the multilingual one.
+const ENGLISH_STATE = "Please refund the duplicate charge on invoice 4411";
+
+describe("lang codes that name no language", () => {
+ const agnostic = ["C", "POSIX", "C.UTF-8", "c.utf8", "c", "posix", "und", "zxx", "mul", "UND", "Zxx", " und "];
+
+ for (const code of agnostic) {
+ it(`${JSON.stringify(code)} abstains and lets detection decide`, () => {
+ expect(englishFromCode(code)).toBeNull();
+ for (const opts of [{ lang: code }, { langGuess: code }]) {
+ const d = new Router().route(ENGLISH_STATE, {}, opts);
+ expect(d.model).toBe("english");
+ expect(d.reason).not.toContain("explicit");
+ expect(d.detection).not.toBeNull();
+ }
+ const installed = new Router({ langGuess: code }).route(ENGLISH_STATE, {});
+ expect(installed.model).toBe("english");
+ expect(installed.detection).not.toBeNull();
+ });
+ }
+
+ it("blank codes still abstain", () => {
+ expect(englishFromCode("")).toBeNull();
+ expect(englishFromCode(null)).toBeNull();
+ expect(englishFromCode(" ")).toBeNull();
+ });
+
+ for (const code of ["en", "eng", "english", "EN", "en-US", "en_US.UTF-8"]) {
+ it(`${code} is still decisive English`, () => {
+ expect(englishFromCode(code)).toBe(true);
+ expect(new Router().route(ENGLISH_STATE, {}, { lang: code }).reason).toContain("explicit");
+ });
+ }
+
+ for (const code of ["de", "fr", "zh", "ja", "pt-BR", "de_DE.UTF-8"]) {
+ it(`${code} is still decisive non-English`, () => {
+ expect(englishFromCode(code)).toBe(false);
+ expect(new Router().route(ENGLISH_STATE, {}, { lang: code }).model).toBe("multilingual");
+ });
+ }
+
+ it("the primary subtag is what is compared", () => {
+ expect(englishFromCode("POSIX-1")).toBeNull();
+ // `ca` (Catalan) and `mn` (Mongolian) share letters with agnostic codes but name languages.
+ expect(englishFromCode("ca")).toBe(false);
+ expect(englishFromCode("mn")).toBe(false);
+ });
+});