Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): count code points in latinProfile like Python
Rahul Ranjan3401424e7f75
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.tsindex bca4e77c..14576726 100644--- a/laya-ts/src/lang.ts+++ b/laya-ts/src/lang.ts@@ -213,6 +213,19 @@ function scriptOf(ch: string): string | null {return null;}+/** Python's `len()`: the number of code points, not of UTF-16 units. Python's `lang.py` divides+ * the diacritic count by `len(lowered)`, and every other port counts code points too (Go+ * `utf8.RuneCountInString`, Rust `chars().count()`, Java `codePointLength`, C++ `RuneCount`), so+ * an astral character must count as one here as well. `String.length` counts it as two, which+ * understated `diacriticRate` on any text carrying emoji -- e.g. `analyse()` on+ * `"é\uD83D\uDE00\uD83D\uDE00" + "-".repeat(46)` read `isEnglish: true` in this port where+ * Python answers `false`, routing the text to the English checkpoint. */+function width(text: string): number {+ let count = 0;+ for (const _ch of text) count++;+ return count;+}+/** String leaves of a state. Keys are ignored: they are usually English field names. */function iterText(state: unknown, depth = 0): string[] {if (depth > 6 || state == null) return [];@@ -350,7 +363,7 @@ export function latinProfile(text: string): LatinProfile {for (const ch of lowered) {if (NON_EN_DIACRITICS.has(ch)) diac += 1;}- const diacRate = diac / Math.max(1, lowered.length);+ const diacRate = diac / Math.max(1, width(lowered));const nonEnglish = diacRate >= NON_EN_DIACRITIC_RATE;const nordicOverlap = words.some((w) => NORDIC_OVERLAP_WORDS.has(w)) &&!words.some((w) => EN_ONLY_WORDS.has(w));@@ -443,12 +456,12 @@ function shouted(text: string): boolean {/** Whether an all-caps line's evidence for `lang` is more than acronym-shaped tokens. */function shoutedEvidence(tokens: string[], lang: string, diacriticRate: number): boolean {- if (!tokens.some((t) => t.length >= SHOUTED_MIN_WORD)) return false;+ if (!tokens.some((t) => width(t) >= SHOUTED_MIN_WORD)) return false;if (diacriticRate >= NON_EN_DIACRITIC_RATE) return true;const stopSet = STOP[lang];if (!stopSet) return false;const matched = new Set(tokens.map((t) => t.toLowerCase()).filter((t) => stopSet.has(t)));- return [...matched].some((w) => w.length >= SHOUTED_MIN_STOPWORD);+ return [...matched].some((w) => width(w) >= SHOUTED_MIN_STOPWORD);}/** Language code for one non-code line, or null when it does not name a foreign language.@@ -586,7 +599,7 @@ function analyseText(text: string): AnalyseResult {// `sag mir das HEUTIGE DATUM` lost its verdict). Diacritic evidence stays measured on the// original text, so a shouted foreign word cannot be blanked out of the safety net.const caps = (text.match(LETTER_RUN_RE) ?? []).filter((m) => m === m.toUpperCase());- if (caps.length > 0 && !caps.some((c) => c.length >= SHOUTED_MIN_WORD)) {+ if (caps.length > 0 && !caps.some((c) => width(c) >= SHOUTED_MIN_WORD)) {const blanked = text.replace(LETTER_RUN_RE, (m) => (m === m.toUpperCase() ? " " : m));if (blanked !== text) lang = latinProfile(blanked).language;}diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.tsindex 5e459b81..4391fdb5 100644--- a/laya-ts/tests/lang.test.ts+++ b/laya-ts/tests/lang.test.ts@@ -37,6 +37,25 @@ describe("lang", () => {expect(a.isEnglish).toBe(false);expect(a.language).toBe(null);});+ // Python divides the diacritic count by len(lowered) -- code points. This port divided by+ // `lowered.length`, which counts UTF-16 units, so every astral character (an emoji, an Old+ // Italic letter) counted as two and understated the rate. The first case sits at the+ // NON_EN_DIACRITIC_RATE threshold: Python's rate is 1/49 = 0.0204 >= 0.02 (non-English, off the+ // English checkpoint), the UTF-16 rate is 1/51 = 0.0196 < 0.02, and `analyse` answered+ // `isEnglish: true` here while Python answered `false` -- two emoji changed the routing verdict.+ // The other two show the rate itself. Expectations from Python's laya.lang.latin_profile on the+ // same inputs.+ it("counts code points, not UTF-16 units, in diacriticRate", () => {+ const flip = analyse("é\uD83D\uDE00\uD83D\uDE00" + "-".repeat(46));+ expect(flip.diacriticRate).toBeCloseTo(0.0204, 6);+ expect(flip.isEnglish).toBe(false);+ const heavy = analyse("créé résumé résumé résumé " + "\uD83D\uDE00".repeat(20));+ expect(heavy.diacriticRate).toBeCloseTo(0.1739, 4);+ expect(heavy.isEnglish).toBe(false);+ const spanish = analyse("día señor niño " + "\uD83D\uDE00".repeat(15));+ expect(spanish.diacriticRate).toBeCloseTo(0.1, 6);+ expect(spanish.isEnglish).toBe(false);+ });// Vietnamese: no stopword list of its own, so NON_EN_DIACRITICS is the only signal that keeps// it off the English checkpoint -- and the tone-marked letters are precomposed codepoints the// set did not hold, so everyday sentences carried a diacritic rate of 0.0 and read as English.