Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(lang): describe the state in analyse, and round rates like Python

cyberspace-cs29b4d2c5997c
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 13e89913..9ec05bbe 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -458,8 +458,25 @@ function nonEnglishSegment(state: unknown, maxChars = 4000): [string, string] |
return null;
}
+// Python's round(x, 4) rounds the exact binary value of the double, breaking a tie to the even
+// digit. Math.round instead rounds `x * 10000`, which differs in two ways: it breaks a tie away
+// from zero, and the multiplication invents a tie where the double is not one. Both are reachable
+// from a ratio of small counts. 1 diacritic in 32 characters is 0.03125 exactly, and Python
+// reports 0.0312 where Math.round reported 0.0313. 1 in 4000 is 0.00025 only to the decimal
+// reader -- the double is 0.0002500000000000000052, above the halfway point, so Python reports
+// 0.0003 while `0.00025 * 10000` lands on 2.5 exactly and a tie rule would report 0.0002. So the
+// digits are taken from the double's own decimal expansion, where "tie" means what it means to
+// Python, and only a real tie consults the parity of the digit before it.
function round4(x: number): number {
- return Math.round(x * 10000) / 10000;
+ const [whole, fraction = ""] = x.toFixed(20).split(".");
+ const kept = Number(whole + fraction.slice(0, 4).padEnd(4, "0"));
+ const rest = fraction.slice(4);
+ const first = rest.charAt(0);
+ const tail = rest.slice(1).replace(/0+$/, "");
+ if (first > "5" || (first === "5" && (tail.length > 0 || kept % 2 === 1))) {
+ return (kept + 1) / 10000;
+ }
+ return kept / 10000;
}
function analyseText(text: string): AnalyseResult {
@@ -577,7 +594,20 @@ export function analyse(state: unknown): AnalyseResult {
best = det;
}
}
- return best ?? result;
+ if (!best) return result;
+ // The value pass answers `language` and `isEnglish` for the state, and leaves the script fields
+ // to the state. Returning `best` whole answered `script`, `scriptProfile` and `nonLatinFraction`
+ // from one leaf instead: a long English note above a short Russian message is 98.6% Latin, and
+ // the result said `script: "cyrillic"` with a profile of `{cyrillic: 1}` -- a description of the
+ // leaf that contradicts `scriptProfile(stateText(state))` for the same input, and that a caller
+ // reading `analyse(state).script` to report the language of a ticket cannot tell from the
+ // `isEnglish` flag beside it. Python's `analyse` overrides the same two fields and no others.
+ return {
+ ...result,
+ language: best.language,
+ isEnglish: false,
+ languageUndecided: best.languageUndecided,
+ };
}
export function isEnglish(state: unknown): boolean {
diff --git a/laya-ts/tests/state-script-fields.test.ts b/laya-ts/tests/state-script-fields.test.ts
new file mode 100644
index 00000000..a2018242
--- /dev/null
+++ b/laya-ts/tests/state-script-fields.test.ts
@@ -0,0 +1,70 @@
+// laya-ts/tests/state-script-fields.test.ts
+import { describe, expect, it } from "vitest";
+import { analyse, detectScript, scriptProfile, stateText } from "../src/lang.js";
+
+// A long English note outnumbers a short message in another script, so the whole-state read is
+// Latin and names no language, and the per-value pass is what flips `isEnglish`. Every expectation
+// below was generated with Python's laya.lang.analyse on the same state.
+const NOTE = "Please read the attached invoice carefully and let me know if you need anything else. ".repeat(20);
+
+describe("analyse: a per-value override keeps the state's own script fields", () => {
+ it.each([
+ ["Мне нужен счет на оплату.", "cyrillic", 0.0141],
+ ["我需要一张发票,请尽快处理,谢谢。", "han", 0.0099],
+ ["χρειάζομαι ένα τιμολόγιο για την παραγγελία", "greek", 0.0264],
+ ["أحتاج إلى فاتورة لهذا الطلب من فضلك", "arabic", 0.0203],
+ ])("reports the state as Latin, not as the %s leaf", (msg, leafScript, share) => {
+ const state = { note: NOTE, msg };
+ const r = analyse(state);
+ // The message still decides the route.
+ expect(r.isEnglish).toBe(false);
+ expect(r.language).toBeNull();
+ // The script fields describe the state, which is overwhelmingly Latin. They used to describe
+ // the leaf instead: `script: leafScript`, `scriptProfile: {[leafScript]: 1}`,
+ // `nonLatinFraction: 1`.
+ expect(r.script).toBe("latin");
+ expect(r.scriptProfile[leafScript]).toBeCloseTo(share, 4);
+ expect(r.nonLatinFraction).toBeCloseTo(share, 4);
+ });
+
+ it("keeps the result self-consistent with scriptProfile(stateText(state))", () => {
+ const state = { note: NOTE, msg: "Мне нужен счет на оплату." };
+ const r = analyse(state);
+ expect(r.scriptProfile).toEqual(scriptProfile(stateText(state)));
+ expect(r.script).toBe(detectScript(stateText(state)));
+ expect(r.nonLatinFraction).toBeCloseTo(1 - (r.scriptProfile["latin"] ?? 0), 4);
+ });
+
+ it("leaves a state that is non-Latin as a whole alone", () => {
+ // Here the whole-state read already names Cyrillic, so the override never runs and the fields
+ // were correct before this change.
+ const r = analyse({ msg: "Мне нужен счет на оплату.", note: "Спасибо" });
+ expect(r.script).toBe("cyrillic");
+ expect(r.isEnglish).toBe(false);
+ });
+});
+
+describe("analyse: the reported rates round the way Python's round() does", () => {
+ // 1 non-English letter in 32 characters is 0.03125 exactly, a tie at four decimals. Python's
+ // round() takes the even digit (0.0312); Math.round took the larger one (0.0313).
+ it("rounds a diacritic rate of 1/32 down", () => {
+ expect(analyse("é" + "a".repeat(31)).diacriticRate).toBe(0.0312);
+ });
+
+ it("rounds a non-Latin fraction of 1/32 down", () => {
+ expect(analyse("你" + "a".repeat(31)).nonLatinFraction).toBe(0.0312);
+ });
+
+ // 1 in 4000 reads as a tie in decimal but is not one in binary: the double is
+ // 0.0002500000000000000052, so Python rounds up. Scaling by 10000 lands on 2.5 exactly and would
+ // round this down, which is why the digits come from the double's own expansion.
+ it("rounds a rate of 1/4000 up, as the double is above the halfway point", () => {
+ expect(analyse("é" + "a".repeat(3999)).diacriticRate).toBe(0.0003);
+ });
+
+ it("still rounds away from a tie the same way as before", () => {
+ // 3 diacritics in 32 characters is 0.09375; the digit before the tie is odd, so both runtimes
+ // round up. This pins the branch that did not change.
+ expect(analyse("ééé" + "a".repeat(29)).diacriticRate).toBe(0.0938);
+ });
+});