Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): port mixed-segment detection from Python analyse() (#384)

Swatantra Yadave37428a124f9
diff --git a/laya-ts/src/lang.ts b/laya-ts/src/lang.ts
index 94fec7b7..6ab5a0af 100644
--- a/laya-ts/src/lang.ts
+++ b/laya-ts/src/lang.ts
@@ -325,6 +325,70 @@ export interface AnalyseResult {
languageUndecided: boolean;
diacriticRate: number;
nonLatinFraction: number;
+ mixedSegment: string | null;
+}
+
+// Code is not prose in any language, but split into words it reads as one: `os.path` is Portuguese
+// (`os`), `round(el, 2)` Spanish (`el`), `non_english` Italian (`non`). A line pasted from a program
+// into an English request must not count as a foreign segment, so a line carrying code syntax --
+// `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers
+// are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.
+const CODE_LINE_RE = /[=;{}[\]]|\w\(/;
+// Slash and backslash compounds are names, not sentences: `Nav/Com` and `OS/2` read as Portuguese
+// (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`). A whitespace token
+// holding a letter or digit, a joiner (`.`, `_`, `/`, `\`) and another letter or digit is an
+// identifier or a compound and is dropped whole.
+const JOINED_RE = /[^\W_][._/\\][^\W_]/;
+// An all-caps token inside mixed-case text is an acronym or a code: `MON`, `LA`, `EST`, `COM`, `DES`
+// are hockey teams, states, time zones and radio bands, not French or Portuguese. A segment written
+// entirely in capitals keeps its words -- a customer shouting in Portuguese is still Portuguese.
+const LETTER_RUN_RE = /[\p{L}]{2,}/gu;
+
+/** Language code for one non-code line, or null when it does not name a foreign language.
+ *
+ * Same evidence bar as `nonEnglishSegment`: four words, a language `latinProfile` will name,
+ * and two *different* words of that language. Acronyms and slash compounds are not words.
+ */
+function namedProseLanguage(segment: string): string | null {
+ if (!segment.trim() || CODE_LINE_RE.test(segment)) return null;
+ const prose = segment.split(/\s+/).filter((tok) => !JOINED_RE.test(tok)).join(" ");
+ // In mixed-case text, replace all-caps runs with spaces (they are acronyms).
+ let cleaned = prose;
+ if ([...prose].some((ch) => /\p{Ll}/u.test(ch))) {
+ cleaned = prose.replace(LETTER_RUN_RE, (m) => (m === m.toUpperCase() ? " " : m));
+ }
+ const tokens = cleaned.match(WORD_RE) ?? [];
+ if (tokens.length < 4) return null;
+ const lang = latinProfile(cleaned).language;
+ if (lang === null || lang === "en") return null;
+ const stopSet = STOP[lang];
+ if (!stopSet) return null;
+ const distinctHits = new Set(tokens.map((w) => w.toLowerCase()).filter((w) => stopSet.has(w)));
+ if (distinctHits.size < 2) return null;
+ return lang;
+}
+
+/** First line or field that, read on its own, is named a non-English language, else null.
+ *
+ * Returns [language, segment]. A segment needs the evidence a whole state needs -- at least four
+ * words, and a language named by `latinProfile` -- and, because one line carries far less text
+ * than a state, two things more: the words that name the language must be two *different* ones
+ * (`COM ... COM` in an English radio listing is one word seen twice), and acronyms and slash
+ * compounds are not words. This adds no new way to call English text foreign; it only stops a
+ * longer English part from outvoting a foreign one. Reads at most `maxChars` characters in all.
+ */
+function nonEnglishSegment(state: unknown, maxChars = 4000): [string, string] | null {
+ let seen = 0;
+ for (const leaf of iterText(state)) {
+ for (const seg of leaf.split("\n")) {
+ if (seen >= maxChars) return null;
+ const capped = seg.slice(0, maxChars - seen);
+ seen += capped.length;
+ const lang = namedProseLanguage(capped);
+ if (lang) return [lang, capped.trim()];
+ }
+ }
+ return null;
}
function round4(x: number): number {
@@ -357,14 +421,14 @@ function analyseText(text: string): AnalyseResult {
return {
script: "unknown", scriptProfile: prof, language: null,
isEnglish: true, languageUndecided: true, diacriticRate: 0.0,
- nonLatinFraction: 0.0,
+ nonLatinFraction: 0.0, mixedSegment: null,
};
}
if (script !== "latin") {
return {
script, scriptProfile: prof, language: null,
isEnglish: false, languageUndecided: true, diacriticRate: 0.0,
- nonLatinFraction: nonLatin,
+ nonLatinFraction: nonLatin, mixedSegment: null,
};
}
const profLat = latinProfile(text);
@@ -375,7 +439,7 @@ function analyseText(text: string): AnalyseResult {
script: "latin", scriptProfile: prof, language: lang,
isEnglish: english, languageUndecided: undecided,
diacriticRate: round4(profLat.diacriticRate),
- nonLatinFraction: nonLatin,
+ nonLatinFraction: nonLatin, mixedSegment: null,
};
}
@@ -409,6 +473,29 @@ export function analyse(state: unknown): AnalyseResult {
// note fill the window, or outvote a short German message, and that message was
// then sent to the English checkpoint.
const result = analyseText(stateText(state));
+ if (result.script === "latin" && result.isEnglish) {
+ // A Portuguese ticket with an English stack trace, error payload or form template reads as
+ // English as a whole, because the English part is longer -- yet the part a question is about
+ // is the customer's, and the English checkpoint cannot read it. So a state that would go to
+ // English is checked line by line and field by field.
+ const leaves = iterText(state);
+ // a single line has no other part to be outvoted by, and was just read whole
+ if (leaves.length > 1 || leaves.some((leaf) => leaf.includes("\n"))) {
+ const found = nonEnglishSegment(state);
+ if (found) {
+ const [lang, mixed] = found;
+ return {
+ ...result,
+ language: lang,
+ isEnglish: false,
+ languageUndecided: false,
+ mixedSegment: mixed,
+ };
+ }
+ }
+ }
+ // A plain string was just read whole. A structured state can still hide a message past the
+ // segment cap, or in a script `latinProfile` does not name.
if (typeof state === "string" || state == null || state instanceof Uint8Array || !result.isEnglish) {
return result;
}
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts
index 0f09b078..09bf951d 100644
--- a/laya-ts/src/router.ts
+++ b/laya-ts/src/router.ts
@@ -444,7 +444,11 @@ export class Router extends HookRegistry {
"the English checkpoint cannot read it";
} else if (!det.isEnglish) {
key = "multilingual";
- if (det.language) {
+ if (det.mixedSegment) {
+ reason =
+ `Latin script, mostly English, but a line or field reads as ${JSON.stringify(det.language)} ` +
+ `(${JSON.stringify(det.mixedSegment.slice(0, 60))}); the English checkpoint cannot read it`;
+ } else if (det.language) {
reason = `Latin script but language looks like ${JSON.stringify(det.language)}, not English`;
} else {
reason =
diff --git a/laya-ts/tests/lang.test.ts b/laya-ts/tests/lang.test.ts
index 342eabab..3ba897b6 100644
--- a/laya-ts/tests/lang.test.ts
+++ b/laya-ts/tests/lang.test.ts
@@ -110,4 +110,86 @@ describe("lang", () => {
expect(buried.isEnglish).toBe(false);
expect(buried.language).toBe("de");
});
+
+ // mixed-segment detection (port of Python's _non_english_segment)
+ // A foreign line embedded in an English ticket must flip isEnglish and record the segment.
+ const TRACE = "I checked the refund status and here is what I found for the customer.\n" +
+ "The charge cleared last Tuesday and the bank confirmed it went through.\n" +
+ "Quero o meu dinheiro de volta agora mesmo porque já esperei demais\n" +
+ "Please escalate to the billing manager if the refund has not been issued by Friday.";
+
+ it("mixed-segment: Portuguese line in English ticket is not english", () => {
+ const a = analyse(TRACE);
+ expect(a.isEnglish).toBe(false);
+ expect(a.language).toBe("pt");
+ expect(a.mixedSegment).toBe(
+ "Quero o meu dinheiro de volta agora mesmo porque já esperei demais",
+ );
+ });
+
+ it("mixed-segment: German error payload in English ticket", () => {
+ const state = "The integration test failed with this error on the staging server " +
+ "and I am not sure whether it is a data issue or a code regression.\n" +
+ "Fehler: Die Verbindung zum Server wurde unterbrochen, bitte versuchen Sie es spaeter noch einmal";
+ const a = analyse(state);
+ expect(a.isEnglish).toBe(false);
+ expect(a.mixedSegment).toContain("Verbindung");
+ });
+
+ it("mixed-segment: Spanish error in structured state", () => {
+ const state = {
+ subject: "Payment failed for a customer in Madrid",
+ description: "The customer tried three times with the same card and each attempt was declined by " +
+ "the gateway, so we would like to know whether the problem is on our side or with the bank.",
+ error: {
+ code: "card_declined",
+ message: "La tarjeta fue rechazada por el banco emisor, contacte con su banco",
+ },
+ };
+ const a = analyse(state);
+ expect(a.isEnglish).toBe(false);
+ expect(a.mixedSegment).toContain("tarjeta");
+ });
+
+ it("mixed-segment: all-caps Portuguese line", () => {
+ const state =
+ "This is the fourth email I have sent about the same order and nobody has answered any of them.\n" +
+ "The customer wrote this in the chat and then closed the window:\n" +
+ "QUERO MEU DINHEIRO DE VOLTA AGORA\n" +
+ "Could someone from the billing team look at order 5512 today?";
+ const a = analyse(state);
+ expect(a.isEnglish).toBe(false);
+ expect(a.mixedSegment).toBe("QUERO MEU DINHEIRO DE VOLTA AGORA");
+ });
+
+ // English stays English: multi-line, short foreign sign-off, code lines
+ it.each([
+ ["multi-line english", "Hi team,\nThe export failed again last night.\nCan you check the logs?\nThanks"],
+ ["short portuguese sign-off", "Please resend the invoice for March, the amount is wrong.\nAtenciosamente, Joao"],
+ ["os.path", "The build broke after the refactor.\nREPO = os.path.dirname(os.path.dirname(__file__))\n" +
+ "Please take a look at the import paths when you can."],
+ ["round(el)", "The latency script crashes on large runs.\nmix[key] = {\"total_s\": round(el, 2)}\n" +
+ "Can you check why the stream is empty?"],
+ ["same word twice (Nav/Com)", "I'm looking for good deals on the following (used or new):\n" +
+ "Aviation Headsets (with mic).\n" +
+ "Handheld Nav/Com tranciever (may consider COM only).\nPortable GPS or Loran Navigator."],
+ ["team codes", "Round two predictions for the pool, as promised.\nQUE vs MON: MON in 7.\n" +
+ "PIT vs NYI: PIT in 5."],
+ ["slash compound", "I need a converter for these image formats.\n" +
+ "DOS, OS/2 or platform independent programs if possible.\nThanks in advance."],
+ ["backslash path", "My modem stopped answering after the upgrade.\nC:\\DOS\\mode COM1:9600,n,8,1,p\n" +
+ "Is that the right line for a 9600 baud connection?"],
+ ])("mixed-segment english stays english: %s", (_label, state) => {
+ expect(analyse(state).isEnglish).toBe(true);
+ expect(analyse(state).mixedSegment).toBeNull();
+ });
+
+ // The segment check reads at most 4000 chars, so a foreign line past that cap is not seen.
+ it("mixed-segment: segment check reads at most the cap", () => {
+ const state = {
+ log: "The export failed again last night for the whole region. ".repeat(80),
+ body: "Quero cancelar meu plano agora mesmo",
+ };
+ expect(analyse(state).mixedSegment).toBeNull();
+ });
});