Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(java): port the shouted-acronym bar, which the fixture already expected

Adeshina Lasisiae63cf646045
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java
index 7e31ee7b..b85a2a7f 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java
@@ -63,6 +63,33 @@ public final class LanguageDetection {
/** The Swedish phrase openers {@code latinProfile} accepts a short login request on. */
private static final Set<String> SWEDISH_PHRASE_OPENERS = Set.of("kan", "jag", "vi");
+ /**
+ * How many letters a matched stopword needs before an all-caps line is believed.
+ *
+ * <p>{@code LOS}, {@code LAS} and {@code EL} are three, three and two letters, so any bar of
+ * four or more rejects them. The reference's sweep over 500,000 acronym runs and 50,000
+ * upper-cased US address lines reports that a bar of three still reads 3,751 acronym runs and
+ * 109 address lines as prose, while four closes both columns outright and five buys nothing
+ * measurable at the cost of 1,860 more real lines. Non-English diacritics are accepted in
+ * place of a long stopword, because the module already treats them as non-English evidence
+ * ({@link LanguageTables#NON_EN_DIACRITIC_RATE}) and neither an acronym nor a US place name
+ * carries any.
+ */
+ private static final int SHOUTED_MIN_STOPWORD = 4;
+
+ /**
+ * How many letters make a token a word rather than an acronym.
+ *
+ * <p>Used twice -- on an all-caps line, by {@link #shoutedEvidence}, and in
+ * {@link #analyseText} to decide whether an all-caps run inside mixed-case text is an acronym
+ * worth blanking. The reference states plainly that the sweep above does not justify this bar
+ * on the committed corpora -- at a stopword bar of four both false-positive columns are
+ * already zero -- and keeps it because the larger uncommitted corpora it was first chosen
+ * against did report acronym false positives surviving a stopword bar of four. It is costly:
+ * 2,198 of 10,960 shouted-prose lines are given up, 82.62% of them six tokens or longer.
+ */
+ private static final int SHOUTED_MIN_WORD = 5;
+
private LanguageDetection() {
}
@@ -283,6 +310,21 @@ public final class LanguageDetection {
return LanguageTables.lookupScriptIndex(cp);
}
+ /**
+ * Whether the non-Latin letters of a Latin-plurality text are enough to carry the state.
+ *
+ * <p>Named here rather than left inline, because the caps bars of {@link #analyseText} ask the
+ * same question for the opposite reason: a state this far from Latin is kept out of them.
+ *
+ * @param nonLatin the non-Latin share of the alphabetic characters
+ * @param nonLatinLetters how many letters that share works out to
+ */
+ static boolean nonLatinCarries(double nonLatin, int nonLatinLetters) {
+ return nonLatin >= LanguageTables.NON_LATIN_FRACTION
+ || (nonLatin >= LanguageTables.NON_LATIN_MIN_FRACTION
+ && nonLatinLetters >= LanguageTables.NON_LATIN_MIN_LETTERS);
+ }
+
/**
* Non-Latin runs that read as words rather than as annotation inside English prose.
*
@@ -488,8 +530,13 @@ public final class LanguageDetection {
* <p>An all-caps token inside mixed-case text is an acronym or a code: {@code MON}, {@code LA},
* {@code EST}, {@code COM}, {@code DES} are hockey teams, states, time zones and radio bands,
* not French or Portuguese. A segment written entirely in capitals keeps its words -- a
- * customer shouting in Portuguese is still Portuguese -- which is why the caller only applies
- * this when the segment holds a lowercase letter.
+ * customer shouting in Portuguese is still Portuguese -- which is why the callers apply this
+ * only to text that is not {@link #shouted}, and hold the shouted case to
+ * {@link #shoutedEvidence} instead.
+ *
+ * <p>This blanks <em>every</em> all-caps run, which is what the reference records as a helper
+ * and what the fixture pins. {@link #analyseText} needs the narrower rule -- only runs short
+ * enough to be acronyms -- and filters before calling, rather than this taking a flag.
*/
static String blankUpperRuns(String text) {
StringBuilder out = new StringBuilder(text.length());
@@ -521,6 +568,42 @@ public final class LanguageDetection {
return out.toString();
}
+ /**
+ * The all-caps runs {@link #blankUpperRuns} would blank, in order.
+ *
+ * <p>The reference reads them off the same {@code finditer} it substitutes with, so the two
+ * cannot disagree about what a run is. Java has no equivalent of handing one scan to both a
+ * substitution and a match list without either a regex -- whose {@code \\w} is a different
+ * set from Python's -- or a callback, so the scan is written twice and the agreement is
+ * stated here instead: both build maximal runs of {@link UnicodeTables#isWordChar} and keep
+ * the ones of at least two code points that {@link UnicodeTables#isUpperString} accepts.
+ */
+ private static List<String> upperLetterRuns(String text) {
+ List<String> runs = new ArrayList<>();
+ int i = 0;
+ int length = text.length();
+ while (i < length) {
+ int cp = text.codePointAt(i);
+ if (!UnicodeTables.isWordChar(cp)) {
+ i += Character.charCount(cp);
+ continue;
+ }
+ int start = i;
+ while (i < length) {
+ int inner = text.codePointAt(i);
+ if (!UnicodeTables.isWordChar(inner)) {
+ break;
+ }
+ i += Character.charCount(inner);
+ }
+ String run = text.substring(start, i);
+ if (codePointLength(run) >= 2 && UnicodeTables.isUpperString(run)) {
+ runs.add(run);
+ }
+ }
+ return runs;
+ }
+
// ------------------------------------------------------------------ the Latin guess
/**
@@ -673,12 +756,91 @@ public final class LanguageDetection {
return latinProfile(text).language();
}
+ /**
+ * True for a <em>cased</em> segment written entirely in capitals.
+ *
+ * <p>The uppercase half is not redundant. A caseless script -- Devanagari, Bengali, CJK -- has
+ * no lowercase either, so testing only for the absence of lowercase would call every such
+ * segment shouted and hold it to a bar it cannot clear: the word scan splits at every
+ * combining mark, so the tokens are short by construction. Nothing is routed on that path
+ * today (the Bengali stopwords are romanised, so a Bengali-script line is named by script and
+ * never reaches here), but the bar belongs to cased text and saying so here keeps it that way.
+ *
+ * <p>That argument covers <em>pure</em> caseless text, which never reaches a caps bar at all.
+ * Text that merely contains some has no lowercase either and does reach one;
+ * {@link #analyseText} keeps both bars off it, because a bar that works by discarding Latin
+ * evidence has nothing to say about the half of the state it cannot read.
+ *
+ * <p>Both halves go through the recorded case tables rather than {@code Character.isUpperCase}
+ * and {@code Character.isLowerCase}: Python's {@code isupper} and {@code islower} are the
+ * general categories plus the {@code Other_Uppercase} and {@code Other_Lowercase} properties,
+ * which the JDK's two methods do not agree with on every code point.
+ */
+ static boolean shouted(String text) {
+ boolean sawUpper = false;
+ int i = 0;
+ while (i < text.length()) {
+ int codePoint = text.codePointAt(i);
+ i += Character.charCount(codePoint);
+ if (UnicodeTables.isLower(codePoint)) {
+ return false;
+ }
+ if (UnicodeTables.isUpper(codePoint)) {
+ sawUpper = true;
+ }
+ }
+ return sawUpper;
+ }
+
+ /**
+ * Whether an all-caps line's evidence for {@code language} is more than acronym-shaped tokens.
+ *
+ * <p>A word of {@link #SHOUTED_MIN_WORD} letters is required in every case, and then either
+ * non-English diacritics or a matched stopword of {@link #SHOUTED_MIN_STOPWORD} letters.
+ * Shouting is not a foreign language: {@code MON}, {@code LA}, {@code EST}, {@code COM} and
+ * {@code DES} are hockey teams, states, time zones and radio bands that collide with French
+ * stopwords, and nothing but their length tells them apart from words.
+ *
+ * @param tokens the word-scan tokens of the line
+ * @param language the language the Latin guess named
+ * @param diacriticRate the rate measured on the line
+ */
+ static boolean shoutedEvidence(List<String> tokens, String language, double diacriticRate) {
+ boolean hasWord = false;
+ for (String token : tokens) {
+ if (codePointLength(token) >= SHOUTED_MIN_WORD) {
+ hasWord = true;
+ break;
+ }
+ }
+ if (!hasWord) {
+ return false;
+ }
+ if (diacriticRate >= LanguageTables.NON_EN_DIACRITIC_RATE) {
+ return true;
+ }
+ Set<String> stop = LanguageTables.STOP_WORDS.get(language);
+ if (stop == null) {
+ // The reference's `_STOP.get(lang, set())` on a language it holds no list for: the
+ // intersection is empty, so no stopword can clear the bar.
+ return false;
+ }
+ for (String token : tokens) {
+ String lowered = pythonLower(token);
+ if (codePointLength(lowered) >= SHOUTED_MIN_STOPWORD && stop.contains(lowered)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Language code for one non-code line, or null when it does not name a foreign language.
*
* <p>Same evidence bar as the segment scan: four words, a language the Latin guess will name,
* and two <em>different</em> words of that language. Acronyms and slash compounds are not
- * words.
+ * words. A segment in all capitals keeps its acronym-shaped tokens -- shouting is not a
+ * foreign language -- so it must also clear {@link #shoutedEvidence}.
*/
static String namedProseLanguage(String segment) {
if (UnicodeTables.isBlank(segment) || hasCodeLine(segment)) {
@@ -691,17 +853,22 @@ public final class LanguageDetection {
}
}
String prose = String.join(" ", kept);
- if (UnicodeTables.hasLower(prose)) {
+ boolean allCaps = shouted(prose);
+ if (!allCaps) {
prose = blankUpperRuns(prose);
}
List<String> tokens = words(prose);
if (tokens.size() < 4) {
return null;
}
- String language = latinProfile(prose).language();
+ LatinProfile profile = latinProfile(prose);
+ String language = profile.language();
if (language == null || "en".equals(language)) {
return null;
}
+ if (allCaps && !shoutedEvidence(tokens, language, profile.diacriticRate())) {
+ return null;
+ }
Set<String> stop = LanguageTables.STOP_WORDS.get(language);
if (stop == null) {
return null;
@@ -763,9 +930,7 @@ public final class LanguageDetection {
: Rounding.round4(1.0 - profile.getOrDefault("latin", 0.0));
int nonLatinLetters = roundHalfEven(nonLatin * countAlpha(text));
if ("latin".equals(script) && !nonLatinWords(text).isEmpty()
- && (nonLatin >= LanguageTables.NON_LATIN_FRACTION
- || (nonLatin >= LanguageTables.NON_LATIN_MIN_FRACTION
- && nonLatinLetters >= LanguageTables.NON_LATIN_MIN_LETTERS))) {
+ && nonLatinCarries(nonLatin, nonLatinLetters)) {
// Non-Latin text is not for the English checkpoint even when Latin letters are the
// plurality: a brand name or order code outvotes the CJK request around it letter for
// letter, though one CJK character carries far more than a letter.
@@ -778,14 +943,77 @@ public final class LanguageDetection {
return new Analysis(script, profile, null, false, true, 0.0, nonLatin, null);
}
LatinProfile latin = latinProfile(text);
+ String language = latin.language();
+ // The acronym bar belongs here too, not only in the segment scan. The scan runs only once
+ // a state already reads English overall, so a state that is *nothing but* an acronym line
+ // -- or one diluted by fewer English lines than it takes to tip this verdict -- never
+ // reached it and was named foreign outright: "MON DES EST LA" routed multilingual on its
+ // own, and so did the same line under one or two lines of English. Vetoing here leaves
+ // the text undecided, so looksNonEnglish still decides it: a shouted line with
+ // non-English diacritics is kept.
+ //
+ // Neither bar applies when the state is carried by a caseless script. Both of them answer
+ // the verdict by taking evidence *away* from the Latin letters, and looksNonEnglish, which
+ // decides what is left, reads Latin diacritics only -- it cannot see the caseless half.
+ // Nothing else catches the fall, either: the script promotion above needs nonLatinWords,
+ // which drops any run starting with a capital, so a shouted mixed-script line has no
+ // promotable word by construction. "PRIVET MON DES EST LA" written in Cyrillic is 35%
+ // Cyrillic letters and was being sent to the English checkpoint on the strength of
+ // disbelieving "MON DES EST LA". Whatever the Latin part is worth, text this far from
+ // Latin is not English, so there is nothing for either bar to buy here and a wrong
+ // language name costs nothing: the checkpoint is chosen on is_english.
+ if (language != null && !"en".equals(language)
+ && !nonLatinCarries(nonLatin, nonLatinLetters)) {
+ if (shouted(text)) {
+ if (!shoutedEvidence(words(text), language, latin.diacriticRate())) {
+ language = null;
+ }
+ } else {
+ // Mixed-case text: the segment scan has always held that an acronym is not a
+ // word, but the whole-state verdict never applied that rule, so the acronyms
+ // voted in it. That is what let a short English state be outvoted -- one or two
+ // lines of English above "MON DES EST LA" still read as French overall, and only
+ // at three did English win the margin. Re-take the verdict without the all-caps
+ // runs. looksNonEnglish and the reported diacritic rate stay measured on the
+ // original text, so a foreign word that happens to be shouted cannot be blanked
+ // out of the diacritic safety net.
+ //
+ // Only *acronym-shaped* runs are blanked. Blanking every all-caps run deleted
+ // emphasis capitals, which are ordinary in real prose and are usually on the
+ // words that carry the sentence, so when the shouted words were the ones that
+ // named the language the evidence simply went: "sag mir das HEUTIGE DATUM" and
+ // "quiero cancelar mi PEDIDO POR FAVOR" both lost their verdict and routed
+ // english. It also broke monotonicity -- the fully upper-cased form held the
+ // shouted bar, but adding one lowercase English token moved the text to this
+ // branch, where it had no bar at all. So a run is blanked only while every one of
+ // them is short enough to be an acronym, by shoutedEvidence's own first test: a
+ // run of SHOUTED_MIN_WORD letters is a word, and a word is not blanked out of its
+ // own sentence. Measured over the 20,708 Latin-locale MASSIVE test rows,
+ // emphasis-capitalised two ways: regressions 1,753 -> 293 and 3,835 -> 752.
+ List<String> caps = upperLetterRuns(text);
+ boolean allAcronyms = !caps.isEmpty();
+ for (String run : caps) {
+ if (codePointLength(run) >= SHOUTED_MIN_WORD) {
+ allAcronyms = false;
+ break;
+ }
+ }
+ if (allAcronyms) {
+ String blanked = blankUpperRuns(text);
+ if (!blanked.equals(text)) {
+ language = latinProfile(blanked).language();
+ }
+ }
+ }
+ }
// Undecided is not English. Treating it as English sent every Latin-script language we
// hold no stopwords for to the checkpoint that cannot read it, silently. When nothing
// identifies the language, non-English letters or a shared Swedish-Danish marker can
// still prefer the multilingual checkpoint; text with neither signal still goes to the
// English one.
- boolean undecided = latin.language() == null;
- boolean english = "en".equals(latin.language()) || (undecided && !latin.looksNonEnglish());
- return new Analysis("latin", profile, latin.language(), english, undecided,
+ boolean undecided = language == null;
+ boolean english = "en".equals(language) || (undecided && !latin.looksNonEnglish());
+ return new Analysis("latin", profile, language, english, undecided,
Rounding.round4(latin.diacriticRate()), nonLatin, null);
}
@@ -846,20 +1074,33 @@ public final class LanguageDetection {
if (detected.english()) {
continue;
}
+ // A named language still has to survive the prose scan: acronyms and slash compounds
+ // are not words. But a veto is not a verdict of English. Dropping the line here
+ // skipped the diacritic branch below, which exists for exactly this -- text that
+ // carries non-English letters and that no stopword list can name. An upper-cased
+ // German question with umlauts was vetoed for having no long stopword and then thrown
+ // away, so a German field routed english; now it falls through and its umlauts carry
+ // it. That is also why languageUndecided is no longer required below: a language named
+ // and then disbelieved is in the same evidential position as one never named.
+ //
+ // The ASCII spelling of that same question is a different case and is not fixed here.
+ // It has no diacritics to fall through to, and its longest matched German stopword is
+ // three letters, so SHOUTED_MIN_STOPWORD vetoes it and nothing recovers it: genuine
+ // German on the English checkpoint, one of the upper-cased casualties the reference
+ // counts and not an example of what this branch buys.
String language = detected.language();
- if (language != null && !"en".equals(language)) {
- if (namedProseLanguage(sample) == null) {
+ boolean named = language != null && !"en".equals(language)
+ && namedProseLanguage(sample) != null;
+ if (!named) {
+ if (!"latin".equals(detected.script()) && !"unknown".equals(detected.script())) {
+ if (nonLatinWords(sample).isEmpty()
+ || countAlpha(sample) < LanguageTables.NON_LATIN_MIN_LETTERS) {
+ continue;
+ }
+ } else if (!(detected.diacriticRate() >= LanguageTables.NON_EN_DIACRITIC_RATE
+ && words(sample).size() >= 4)) {
continue;
}
- } else if (!"latin".equals(detected.script()) && !"unknown".equals(detected.script())) {
- if (nonLatinWords(sample).isEmpty()
- || countAlpha(sample) < LanguageTables.NON_LATIN_MIN_LETTERS) {
- continue;
- }
- } else if (!(detected.languageUndecided()
- && detected.diacriticRate() >= LanguageTables.NON_EN_DIACRITIC_RATE
- && words(sample).size() >= 4)) {
- continue;
}
int letters = countAlpha(sample);
if (letters > bestLetters) {
diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/ShoutedAcronymBarTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/ShoutedAcronymBarTest.java
new file mode 100644
index 00000000..81f999a5
--- /dev/null
+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/ShoutedAcronymBarTest.java
@@ -0,0 +1,255 @@
+package com.convaiinnovations.laya.lang;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The caps bars, asserted directly rather than through the recorded fixture.
+ *
+ * <p>{@link LanguageDetectionParityTest} already covers every case in {@code lang_detect.json},
+ * but it covers them as data: a rule can be removed and the fixture re-recorded from a Python
+ * reference that lost the same rule, and nothing here would notice. These assertions name the
+ * behaviour instead, so the rule has to be deleted from two places to disappear quietly.
+ *
+ * <p>The three cases the rule exists to balance are all here. An all-caps line of English
+ * abbreviations that collide with French stopwords must not be named French; emphasis capitals in
+ * genuine prose must not be blanked out of their own sentence; and a state carried by a caseless
+ * script must be held to neither bar, because both work by discarding Latin evidence.
+ */
+class ShoutedAcronymBarTest {
+
+ /** English abbreviations -- hockey teams, states, time zones, radio bands -- that read as French. */
+ private static final String ACRONYMS = "MON LA EST COM DES";
+
+ /** A genuinely Portuguese line, written without its accents as a customer would type it. */
+ private static final String PORTUGUESE =
+ "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum";
+
+ /**
+ * The same line shouted, spelled out rather than upper-cased.
+ *
+ * <p>{@code String.toUpperCase()} without a locale is locale-sensitive -- under a Turkish
+ * default it maps {@code i} to U+0130 -- and this line has four of them, so computing it
+ * would make the assertion depend on the machine.
+ */
+ private static final String PORTUGUESE_SHOUTED =
+ "NAO CONSIGO ENTRAR NA MINHA CONTA E A SENHA NAO FUNCIONA DE JEITO NENHUM";
+
+ private static final String ENGLISH_OPENER =
+ "The customer opened this ticket yesterday and we have asked for a screenshot.";
+
+ private static final String ENGLISH_CLOSER =
+ "We will escalate this to the platform team if it is not resolved today.";
+
+ @Test
+ @DisplayName("an all-caps line of English abbreviations does not name French")
+ void acronymLineIsNotProse() {
+ assertNull(LanguageDetection.namedProseLanguage(ACRONYMS),
+ "bare abbreviations are acronym-shaped and name no language");
+ assertEquals("pt", LanguageDetection.namedProseLanguage(PORTUGUESE),
+ "the Portuguese line is still named, so the veto is not simply vetoing everything");
+ }
+
+ @Test
+ @DisplayName("the genuinely foreign line decides a state the acronym line used to win")
+ void foreignLineOutvotesTheAcronymLine() {
+ String state = ENGLISH_OPENER + "\n" + ACRONYMS + "\n" + PORTUGUESE + "\n" + ENGLISH_CLOSER;
+ LanguageDetection.Analysis analysis = LanguageDetection.analyse(state);
+ assertEquals("pt", analysis.language(), "the Portuguese line decides, not the acronyms");
+ assertFalse(analysis.english(), "a Portuguese message is not for the English checkpoint");
+ assertFalse(analysis.languageUndecided(), "the language is named");
+ assertEquals(PORTUGUESE, analysis.mixedSegment(), "and the line that named it is reported");
+ }
+
+ @Test
+ @DisplayName("a state that is nothing but an acronym line is undecided, not French")
+ void acronymOnlyStateIsUndecided() {
+ LanguageDetection.Analysis analysis = LanguageDetection.analyse(ACRONYMS);
+ assertNull(analysis.language(), "the whole-state verdict applies the bar too");
+ assertTrue(analysis.languageUndecided(), "vetoing leaves the text undecided");
+ // Undecided with no non-English letter is English: the veto takes the French verdict
+ // away, it does not replace it with one of its own.
+ assertTrue(analysis.english(), "nothing here is non-English, so the English checkpoint reads it");
+ }
+
+ @Test
+ @DisplayName("one English line above the acronyms no longer routes multilingual")
+ void shortEnglishStateIsNotOutvoted() {
+ LanguageDetection.Analysis analysis =
+ LanguageDetection.analyse("We have asked for a screenshot.\n" + ACRONYMS);
+ assertEquals("en", analysis.language(), "the acronyms no longer vote in the verdict");
+ assertTrue(analysis.english());
+ }
+
+ @Test
+ @DisplayName("emphasis capitals keep the verdict they name")
+ void emphasisCapitalsAreNotBlanked() {
+ // Only acronym-shaped runs are blanked. Blanking every all-caps run deleted the words that
+ // carried these two sentences, and both of them routed english with no language at all.
+ LanguageDetection.Analysis german = LanguageDetection.analyse("sag mir das HEUTIGE DATUM");
+ assertEquals("de", german.language(), "HEUTIGE and DATUM are words, not acronyms");
+ assertFalse(german.english());
+
+ LanguageDetection.Analysis spanish =
+ LanguageDetection.analyse("quiero cancelar mi PEDIDO POR FAVOR");
+ assertEquals("es", spanish.language(), "PEDIDO is a word, so no run is blanked");
+ assertFalse(spanish.english());
+ }
+
+ @Test
+ @DisplayName("an all-caps line with non-English letters clears the bar on its diacritics")
+ void diacriticsStandInForALongStopword() {
+ // Its longest matched German stopword is three letters, so the stopword half of the bar
+ // vetoes it; the umlauts carry it, and the veto leaves it undecided rather than English.
+ LanguageDetection.Analysis analysis = LanguageDetection.analyse("WIE SPÄT IST ES IN KÖLN");
+ assertNull(analysis.language(), "no stopword list can name it");
+ assertTrue(analysis.languageUndecided());
+ assertFalse(analysis.english(), "a veto is not a verdict of English");
+ }
+
+ /**
+ * A German field the whole-state verdict names but the prose scan disbelieves.
+ *
+ * <p>{@code wie} is the only German stopword it matches, and the prose scan wants two
+ * <em>different</em> ones, so it is vetoed -- while the whole-state verdict, which has no such
+ * rule, names it {@code de}. That disagreement is the only thing the restructure below
+ * changes, so a field that merely clears both bars, or fails both, pins nothing.
+ */
+ private static final String VETOED_GERMAN = "wie Überprüfung wie Anmeldung";
+
+ private static final String LONG_ENGLISH_NOTE =
+ "The agent replied to the customer and closed the ticket today, and we have also "
+ + "asked the platform team to confirm that the account is in good standing before "
+ + "we escalate.";
+
+ @Test
+ @DisplayName("a vetoed field still reaches the multilingual checkpoint on its diacritics")
+ void vetoIsNotAVerdictOfEnglishInAField() {
+ Map<String, Object> state = new LinkedHashMap<>();
+ state.put("note", LONG_ENGLISH_NOTE);
+ state.put("msg", VETOED_GERMAN);
+
+ // Three anchors, so a change that makes the assertion below pass for some other reason is
+ // not mistaken for this rule holding. The joined window has to read English -- the note is
+ // long enough to dilute the umlauts below the diacritic floor -- the prose scan has to
+ // veto the field, and the whole-state verdict has to name it anyway. Only then is the
+ // per-field pass the last thing that can catch it, which is the position the fix is about.
+ assertTrue(LanguageDetection.analyseText(LanguageDetection.stateText(state)).english(),
+ "the joined window has to read English, or this case tests nothing");
+ assertNull(LanguageDetection.namedProseLanguage(VETOED_GERMAN),
+ "the prose scan has to veto the field, or this case tests nothing");
+ assertEquals("de", LanguageDetection.analyseText(VETOED_GERMAN).language(),
+ "and the whole-state verdict has to name it, or this case tests nothing");
+ assertNull(LanguageDetection.nonEnglishSegment(state, LanguageDetection.MAX_CHARS),
+ "so the segment scan finds nothing");
+
+ // Before the fix the field was dropped here: a named-then-disbelieved language took the
+ // first branch and never reached the diacritic branch, which exists for exactly this.
+ LanguageDetection.Analysis leaf = LanguageDetection.leafNonEnglish(VETOED_GERMAN);
+ assertNotNull(leaf, "a veto is not a verdict of English");
+ assertFalse(leaf.english());
+
+ LanguageDetection.Analysis analysis = LanguageDetection.analyse(state);
+ assertFalse(analysis.english(),
+ "so the German field pulls the state off the English checkpoint");
+ assertEquals("de", analysis.language());
+ }
+
+ @Test
+ @DisplayName("a shouted field with umlauts is carried by the diacritic branch")
+ void shoutedFieldFallsThroughOnItsDiacritics() {
+ // The same position reached the other way: here the whole-state verdict vetoes the field
+ // too, on the shouted bar, so it arrives undecided rather than named.
+ Map<String, Object> state = new LinkedHashMap<>();
+ state.put("note", LONG_ENGLISH_NOTE);
+ state.put("msg", "WIE SPÄT IST ES IN KÖLN");
+ assertTrue(LanguageDetection.analyseText(LanguageDetection.stateText(state)).english(),
+ "the joined window has to read English, or this case tests nothing");
+ assertFalse(LanguageDetection.analyse(state).english());
+ assertTrue(LanguageDetection.analyse(state).languageUndecided());
+ }
+
+ @Test
+ @DisplayName("a caseless script is held to neither caps bar")
+ void caselessScriptKeepsTheBarsOff() {
+ // 35% Cyrillic letters. Both bars answer the verdict by taking evidence away from the
+ // Latin letters, and what decides what is left reads Latin diacritics only -- it cannot
+ // see the caseless half. Text this far from Latin is not English whatever the Latin part
+ // is worth, so the language name is allowed to stand.
+ LanguageDetection.Analysis analysis =
+ LanguageDetection.analyse("ПРИВЕТ MON DES EST LA");
+ assertEquals("fr", analysis.language(), "the bar is kept off a mostly-non-Latin state");
+ assertFalse(analysis.english(), "which is the only thing that decides the checkpoint");
+ }
+
+ @Test
+ @DisplayName("shouted needs an uppercase character, so caseless text is not shouted")
+ void shoutedNeedsAnUppercaseCharacter() {
+ assertTrue(LanguageDetection.shouted(ACRONYMS));
+ assertFalse(LanguageDetection.shouted("sag mir das HEUTIGE DATUM"), "one lowercase letter is enough");
+ assertFalse(LanguageDetection.shouted("hola"));
+ assertFalse(LanguageDetection.shouted("1234"), "no cased character at all");
+ // Devanagari has no lowercase either; testing only for the absence of lowercase would call
+ // this shouted and hold it to a bar its tokens cannot clear.
+ assertFalse(LanguageDetection.shouted("हिन्दी बोलता"));
+ }
+
+ @Test
+ @DisplayName("shouted evidence needs a long word, plus diacritics or a long stopword")
+ void shoutedEvidenceNeedsMoreThanAcronyms() {
+ assertFalse(LanguageDetection.shoutedEvidence(
+ LanguageDetection.words(ACRONYMS), "fr", 0.0),
+ "every token is acronym-shaped");
+ assertTrue(LanguageDetection.shoutedEvidence(
+ LanguageDetection.words(PORTUGUESE_SHOUTED), "pt", 0.0),
+ "shouted Portuguese is still Portuguese");
+ assertFalse(LanguageDetection.shoutedEvidence(List.of("PARA", "NAO"), "pt", 0.0),
+ "no token reaches the word bar");
+ assertTrue(LanguageDetection.shoutedEvidence(List.of("KOELN", "IST"), "de", 0.05),
+ "non-English letters stand in for a long stopword");
+ assertFalse(LanguageDetection.shoutedEvidence(List.of("SENHA", "NAO"), "zz", 0.0),
+ "a language with no stopword list can match nothing");
+ }
+
+ @Test
+ @DisplayName("both bars are at the value the reference swept to, not one letter lower")
+ void barsSitWhereTheSweepPutThem() {
+ // The acronym line above never reaches the stopword bar -- no token of it is long enough
+ // to clear the word bar first -- so without these two the bars could each be loosened by
+ // a letter with nothing failing. Each input clears one bar and is stopped by the other.
+
+ // PARA and ESTA are four-letter Spanish stopwords: the stopword bar is cleared, and the
+ // word bar is what stops it. At four it would pass.
+ assertFalse(LanguageDetection.shoutedEvidence(
+ LanguageDetection.words("PARA ESTA PARA ESTO"), "es", 0.0),
+ "a four-letter stopword is not a five-letter word");
+
+ // SPAET and KOELN clear the word bar, and the longest German stopword matched is `wie`,
+ // three letters: the stopword bar is what stops it. At three it would pass.
+ assertFalse(LanguageDetection.shoutedEvidence(
+ LanguageDetection.words("WIE SPAET IST ES IN KOELN"), "de", 0.0),
+ "three letters of stopword is not evidence");
+ }
+
+ @Test
+ @DisplayName("the ASCII spelling of that German question is a casualty, and stays one")
+ void asciiGermanIsAKnownCasualty() {
+ // Not an endorsement: the reference counts this among the roughly 3,727 upper-cased rows
+ // the pair of bars gives up, and says plainly that nothing recovers it -- it has no
+ // diacritics to fall through to and no stopword long enough. It is asserted because it is
+ // the cheapest behavioural pin on the stopword bar: lower the bar to three and genuine
+ // German starts routing multilingual here, which is better, but it is not what the merged
+ // reference does, and the Java port is not the place to change the rule.
+ assertTrue(LanguageDetection.analyse("WIE SPAET IST ES IN KOELN").english(),
+ "the bar vetoes it and no diacritic carries it");
+ }
+}