Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): count code points in latinProfile like Python

Rahul Ranjan3401424e7f75
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index bca4e77c..14576726 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -213,6 +213,19 @@ function scriptOf(ch: string): string | null {
return null;
}
+/** Python's `len()`: the number of code points, not of UTF-16 units. Python's `lang.py` divides
+ * the diacritic count by `len(lowered)`, and every other port counts code points too (Go
+ * `utf8.RuneCountInString`, Rust `chars().count()`, Java `codePointLength`, C++ `RuneCount`), so
+ * an astral character must count as one here as well. `String.length` counts it as two, which
+ * understated `diacriticRate` on any text carrying emoji -- e.g. `analyse()` on
+ * `"é\uD83D\uDE00\uD83D\uDE00" + "-".repeat(46)` read `isEnglish: true` in this port where
+ * Python answers `false`, routing the text to the English checkpoint. */
+function width(text: string): number {
+ let count = 0;
+ for (const _ch of text) count++;
+ return count;
+}
+
/** String leaves of a state. Keys are ignored: they are usually English field names. */
function iterText(state: unknown, depth = 0): string[] {
if (depth > 6 || state == null) return [];
@@ -350,7 +363,7 @@ export function latinProfile(text: string): LatinProfile {
for (const ch of lowered) {
if (NON_EN_DIACRITICS.has(ch)) diac += 1;
}
- const diacRate = diac / Math.max(1, lowered.length);
+ const diacRate = diac / Math.max(1, width(lowered));
const nonEnglish = diacRate >= NON_EN_DIACRITIC_RATE;
const nordicOverlap = words.some((w) => NORDIC_OVERLAP_WORDS.has(w)) &&
!words.some((w) => EN_ONLY_WORDS.has(w));
@@ -443,12 +456,12 @@ function shouted(text: string): boolean {
/** Whether an all-caps line's evidence for `lang` is more than acronym-shaped tokens. */
function shoutedEvidence(tokens: string[], lang: string, diacriticRate: number): boolean {
- if (!tokens.some((t) => t.length >= SHOUTED_MIN_WORD)) return false;
+ if (!tokens.some((t) => width(t) >= SHOUTED_MIN_WORD)) return false;
if (diacriticRate >= NON_EN_DIACRITIC_RATE) return true;
const stopSet = STOP[lang];
if (!stopSet) return false;
const matched = new Set(tokens.map((t) => t.toLowerCase()).filter((t) => stopSet.has(t)));
- return [...matched].some((w) => w.length >= SHOUTED_MIN_STOPWORD);
+ return [...matched].some((w) => width(w) >= SHOUTED_MIN_STOPWORD);
}
/** Language code for one non-code line, or null when it does not name a foreign language.
@@ -586,7 +599,7 @@ function analyseText(text: string): AnalyseResult {
// `sag mir das HEUTIGE DATUM` lost its verdict). Diacritic evidence stays measured on the
// original text, so a shouted foreign word cannot be blanked out of the safety net.
const caps = (text.match(LETTER_RUN_RE) ?? []).filter((m) => m === m.toUpperCase());
- if (caps.length > 0 && !caps.some((c) => c.length >= SHOUTED_MIN_WORD)) {
+ if (caps.length > 0 && !caps.some((c) => width(c) >= SHOUTED_MIN_WORD)) {
const blanked = text.replace(LETTER_RUN_RE, (m) => (m === m.toUpperCase() ? " " : m));
if (blanked !== text) lang = latinProfile(blanked).language;
}
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index 5e459b81..4391fdb5 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -37,6 +37,25 @@ describe("lang", () => {
expect(a.isEnglish).toBe(false);
expect(a.language).toBe(null);
});
+ // Python divides the diacritic count by len(lowered) -- code points. This port divided by
+ // `lowered.length`, which counts UTF-16 units, so every astral character (an emoji, an Old
+ // Italic letter) counted as two and understated the rate. The first case sits at the
+ // NON_EN_DIACRITIC_RATE threshold: Python's rate is 1/49 = 0.0204 >= 0.02 (non-English, off the
+ // English checkpoint), the UTF-16 rate is 1/51 = 0.0196 < 0.02, and `analyse` answered
+ // `isEnglish: true` here while Python answered `false` -- two emoji changed the routing verdict.
+ // The other two show the rate itself. Expectations from Python's laya.lang.latin_profile on the
+ // same inputs.
+ it("counts code points, not UTF-16 units, in diacriticRate", () => {
+ const flip = analyse("é\uD83D\uDE00\uD83D\uDE00" + "-".repeat(46));
+ expect(flip.diacriticRate).toBeCloseTo(0.0204, 6);
+ expect(flip.isEnglish).toBe(false);
+ const heavy = analyse("créé résumé résumé résumé " + "\uD83D\uDE00".repeat(20));
+ expect(heavy.diacriticRate).toBeCloseTo(0.1739, 4);
+ expect(heavy.isEnglish).toBe(false);
+ const spanish = analyse("día señor niño " + "\uD83D\uDE00".repeat(15));
+ expect(spanish.diacriticRate).toBeCloseTo(0.1, 6);
+ expect(spanish.isEnglish).toBe(false);
+ });
// Vietnamese: no stopword list of its own, so NON_EN_DIACRITICS is the only signal that keeps
// it off the English checkpoint -- and the tone-marked letters are precomposed codepoints the
// set did not hold, so everyday sentences carried a diacritic rate of 0.0 and read as English.