Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(lang): describe the state in analyse, and round rates like Python
cyberspace-cs29b4d2c5997c
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex 13e89913..9ec05bbe 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -458,8 +458,25 @@ function nonEnglishSegment(state: unknown, maxChars = 4000): [string, string] |return null;}+// Python's round(x, 4) rounds the exact binary value of the double, breaking a tie to the even+// digit. Math.round instead rounds `x * 10000`, which differs in two ways: it breaks a tie away+// from zero, and the multiplication invents a tie where the double is not one. Both are reachable+// from a ratio of small counts. 1 diacritic in 32 characters is 0.03125 exactly, and Python+// reports 0.0312 where Math.round reported 0.0313. 1 in 4000 is 0.00025 only to the decimal+// reader -- the double is 0.0002500000000000000052, above the halfway point, so Python reports+// 0.0003 while `0.00025 * 10000` lands on 2.5 exactly and a tie rule would report 0.0002. So the+// digits are taken from the double's own decimal expansion, where "tie" means what it means to+// Python, and only a real tie consults the parity of the digit before it.function round4(x: number): number {- return Math.round(x * 10000) / 10000;+ const [whole, fraction = ""] = x.toFixed(20).split(".");+ const kept = Number(whole + fraction.slice(0, 4).padEnd(4, "0"));+ const rest = fraction.slice(4);+ const first = rest.charAt(0);+ const tail = rest.slice(1).replace(/0+$/, "");+ if (first > "5" || (first === "5" && (tail.length > 0 || kept % 2 === 1))) {+ return (kept + 1) / 10000;+ }+ return kept / 10000;}function analyseText(text: string): AnalyseResult {@@ -577,7 +594,20 @@ export function analyse(state: unknown): AnalyseResult {best = det;}}- return best ?? result;+ if (!best) return result;+ // The value pass answers `language` and `isEnglish` for the state, and leaves the script fields+ // to the state. Returning `best` whole answered `script`, `scriptProfile` and `nonLatinFraction`+ // from one leaf instead: a long English note above a short Russian message is 98.6% Latin, and+ // the result said `script: "cyrillic"` with a profile of `{cyrillic: 1}` -- a description of the+ // leaf that contradicts `scriptProfile(stateText(state))` for the same input, and that a caller+ // reading `analyse(state).script` to report the language of a ticket cannot tell from the+ // `isEnglish` flag beside it. Python's `analyse` overrides the same two fields and no others.+ return {+ ...result,+ language: best.language,+ isEnglish: false,+ languageUndecided: best.languageUndecided,+ };}export function isEnglish(state: unknown): boolean {diff --git a/laya-ts/tests/state-script-fields.test.ts b/laya-ts/tests/state-script-fields.test.tsnew file mode 100644index 00000000..a2018242--- /dev/null+++ b/laya-ts/tests/state-script-fields.test.ts@@ -0,0 +1,70 @@+// laya-ts/tests/state-script-fields.test.ts+import { describe, expect, it } from "vitest";+import { analyse, detectScript, scriptProfile, stateText } from "../src/lang.js";++// A long English note outnumbers a short message in another script, so the whole-state read is+// Latin and names no language, and the per-value pass is what flips `isEnglish`. Every expectation+// below was generated with Python's laya.lang.analyse on the same state.+const NOTE = "Please read the attached invoice carefully and let me know if you need anything else. ".repeat(20);++describe("analyse: a per-value override keeps the state's own script fields", () => {+ it.each([+ ["Мне нужен счет на оплату.", "cyrillic", 0.0141],+ ["我需要一张发票,请尽快处理,谢谢。", "han", 0.0099],+ ["χρειάζομαι ένα τιμολόγιο για την παραγγελία", "greek", 0.0264],+ ["أحتاج إلى فاتورة لهذا الطلب من فضلك", "arabic", 0.0203],+ ])("reports the state as Latin, not as the %s leaf", (msg, leafScript, share) => {+ const state = { note: NOTE, msg };+ const r = analyse(state);+ // The message still decides the route.+ expect(r.isEnglish).toBe(false);+ expect(r.language).toBeNull();+ // The script fields describe the state, which is overwhelmingly Latin. They used to describe+ // the leaf instead: `script: leafScript`, `scriptProfile: {[leafScript]: 1}`,+ // `nonLatinFraction: 1`.+ expect(r.script).toBe("latin");+ expect(r.scriptProfile[leafScript]).toBeCloseTo(share, 4);+ expect(r.nonLatinFraction).toBeCloseTo(share, 4);+ });++ it("keeps the result self-consistent with scriptProfile(stateText(state))", () => {+ const state = { note: NOTE, msg: "Мне нужен счет на оплату." };+ const r = analyse(state);+ expect(r.scriptProfile).toEqual(scriptProfile(stateText(state)));+ expect(r.script).toBe(detectScript(stateText(state)));+ expect(r.nonLatinFraction).toBeCloseTo(1 - (r.scriptProfile["latin"] ?? 0), 4);+ });++ it("leaves a state that is non-Latin as a whole alone", () => {+ // Here the whole-state read already names Cyrillic, so the override never runs and the fields+ // were correct before this change.+ const r = analyse({ msg: "Мне нужен счет на оплату.", note: "Спасибо" });+ expect(r.script).toBe("cyrillic");+ expect(r.isEnglish).toBe(false);+ });+});++describe("analyse: the reported rates round the way Python's round() does", () => {+ // 1 non-English letter in 32 characters is 0.03125 exactly, a tie at four decimals. Python's+ // round() takes the even digit (0.0312); Math.round took the larger one (0.0313).+ it("rounds a diacritic rate of 1/32 down", () => {+ expect(analyse("é" + "a".repeat(31)).diacriticRate).toBe(0.0312);+ });++ it("rounds a non-Latin fraction of 1/32 down", () => {+ expect(analyse("你" + "a".repeat(31)).nonLatinFraction).toBe(0.0312);+ });++ // 1 in 4000 reads as a tie in decimal but is not one in binary: the double is+ // 0.0002500000000000000052, so Python rounds up. Scaling by 10000 lands on 2.5 exactly and would+ // round this down, which is why the digits come from the double's own expansion.+ it("rounds a rate of 1/4000 up, as the double is above the halfway point", () => {+ expect(analyse("é" + "a".repeat(3999)).diacriticRate).toBe(0.0003);+ });++ it("still rounds away from a tie the same way as before", () => {+ // 3 diacritics in 32 characters is 0.09375; the digit before the tie is odd, so both runtimes+ // round up. This pins the branch that did not change.+ expect(analyse("ééé" + "a".repeat(29)).diacriticRate).toBe(0.0938);+ });+});