Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(java): port the shouted-acronym bar, which the fixture already expected
Adeshina Lasisiae63cf646045
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.javaindex 7e31ee7b..b85a2a7f 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageDetection.java@@ -63,6 +63,33 @@ public final class LanguageDetection {/** The Swedish phrase openers {@code latinProfile} accepts a short login request on. */private static final Set<String> SWEDISH_PHRASE_OPENERS = Set.of("kan", "jag", "vi");+ /**+ * How many letters a matched stopword needs before an all-caps line is believed.+ *+ * <p>{@code LOS}, {@code LAS} and {@code EL} are three, three and two letters, so any bar of+ * four or more rejects them. The reference's sweep over 500,000 acronym runs and 50,000+ * upper-cased US address lines reports that a bar of three still reads 3,751 acronym runs and+ * 109 address lines as prose, while four closes both columns outright and five buys nothing+ * measurable at the cost of 1,860 more real lines. Non-English diacritics are accepted in+ * place of a long stopword, because the module already treats them as non-English evidence+ * ({@link LanguageTables#NON_EN_DIACRITIC_RATE}) and neither an acronym nor a US place name+ * carries any.+ */+ private static final int SHOUTED_MIN_STOPWORD = 4;++ /**+ * How many letters make a token a word rather than an acronym.+ *+ * <p>Used twice -- on an all-caps line, by {@link #shoutedEvidence}, and in+ * {@link #analyseText} to decide whether an all-caps run inside mixed-case text is an acronym+ * worth blanking. The reference states plainly that the sweep above does not justify this bar+ * on the committed corpora -- at a stopword bar of four both false-positive columns are+ * already zero -- and keeps it because the larger uncommitted corpora it was first chosen+ * against did report acronym false positives surviving a stopword bar of four. It is costly:+ * 2,198 of 10,960 shouted-prose lines are given up, 82.62% of them six tokens or longer.+ */+ private static final int SHOUTED_MIN_WORD = 5;+private LanguageDetection() {}@@ -283,6 +310,21 @@ public final class LanguageDetection {return LanguageTables.lookupScriptIndex(cp);}+ /**+ * Whether the non-Latin letters of a Latin-plurality text are enough to carry the state.+ *+ * <p>Named here rather than left inline, because the caps bars of {@link #analyseText} ask the+ * same question for the opposite reason: a state this far from Latin is kept out of them.+ *+ * @param nonLatin the non-Latin share of the alphabetic characters+ * @param nonLatinLetters how many letters that share works out to+ */+ static boolean nonLatinCarries(double nonLatin, int nonLatinLetters) {+ return nonLatin >= LanguageTables.NON_LATIN_FRACTION+ || (nonLatin >= LanguageTables.NON_LATIN_MIN_FRACTION+ && nonLatinLetters >= LanguageTables.NON_LATIN_MIN_LETTERS);+ }+/*** Non-Latin runs that read as words rather than as annotation inside English prose.*@@ -488,8 +530,13 @@ public final class LanguageDetection {* <p>An all-caps token inside mixed-case text is an acronym or a code: {@code MON}, {@code LA},* {@code EST}, {@code COM}, {@code DES} are hockey teams, states, time zones and radio bands,* not French or Portuguese. A segment written entirely in capitals keeps its words -- a- * customer shouting in Portuguese is still Portuguese -- which is why the caller only applies- * this when the segment holds a lowercase letter.+ * customer shouting in Portuguese is still Portuguese -- which is why the callers apply this+ * only to text that is not {@link #shouted}, and hold the shouted case to+ * {@link #shoutedEvidence} instead.+ *+ * <p>This blanks <em>every</em> all-caps run, which is what the reference records as a helper+ * and what the fixture pins. {@link #analyseText} needs the narrower rule -- only runs short+ * enough to be acronyms -- and filters before calling, rather than this taking a flag.*/static String blankUpperRuns(String text) {StringBuilder out = new StringBuilder(text.length());@@ -521,6 +568,42 @@ public final class LanguageDetection {return out.toString();}+ /**+ * The all-caps runs {@link #blankUpperRuns} would blank, in order.+ *+ * <p>The reference reads them off the same {@code finditer} it substitutes with, so the two+ * cannot disagree about what a run is. Java has no equivalent of handing one scan to both a+ * substitution and a match list without either a regex -- whose {@code \\w} is a different+ * set from Python's -- or a callback, so the scan is written twice and the agreement is+ * stated here instead: both build maximal runs of {@link UnicodeTables#isWordChar} and keep+ * the ones of at least two code points that {@link UnicodeTables#isUpperString} accepts.+ */+ private static List<String> upperLetterRuns(String text) {+ List<String> runs = new ArrayList<>();+ int i = 0;+ int length = text.length();+ while (i < length) {+ int cp = text.codePointAt(i);+ if (!UnicodeTables.isWordChar(cp)) {+ i += Character.charCount(cp);+ continue;+ }+ int start = i;+ while (i < length) {+ int inner = text.codePointAt(i);+ if (!UnicodeTables.isWordChar(inner)) {+ break;+ }+ i += Character.charCount(inner);+ }+ String run = text.substring(start, i);+ if (codePointLength(run) >= 2 && UnicodeTables.isUpperString(run)) {+ runs.add(run);+ }+ }+ return runs;+ }+// ------------------------------------------------------------------ the Latin guess/**@@ -673,12 +756,91 @@ public final class LanguageDetection {return latinProfile(text).language();}+ /**+ * True for a <em>cased</em> segment written entirely in capitals.+ *+ * <p>The uppercase half is not redundant. A caseless script -- Devanagari, Bengali, CJK -- has+ * no lowercase either, so testing only for the absence of lowercase would call every such+ * segment shouted and hold it to a bar it cannot clear: the word scan splits at every+ * combining mark, so the tokens are short by construction. Nothing is routed on that path+ * today (the Bengali stopwords are romanised, so a Bengali-script line is named by script and+ * never reaches here), but the bar belongs to cased text and saying so here keeps it that way.+ *+ * <p>That argument covers <em>pure</em> caseless text, which never reaches a caps bar at all.+ * Text that merely contains some has no lowercase either and does reach one;+ * {@link #analyseText} keeps both bars off it, because a bar that works by discarding Latin+ * evidence has nothing to say about the half of the state it cannot read.+ *+ * <p>Both halves go through the recorded case tables rather than {@code Character.isUpperCase}+ * and {@code Character.isLowerCase}: Python's {@code isupper} and {@code islower} are the+ * general categories plus the {@code Other_Uppercase} and {@code Other_Lowercase} properties,+ * which the JDK's two methods do not agree with on every code point.+ */+ static boolean shouted(String text) {+ boolean sawUpper = false;+ int i = 0;+ while (i < text.length()) {+ int codePoint = text.codePointAt(i);+ i += Character.charCount(codePoint);+ if (UnicodeTables.isLower(codePoint)) {+ return false;+ }+ if (UnicodeTables.isUpper(codePoint)) {+ sawUpper = true;+ }+ }+ return sawUpper;+ }++ /**+ * Whether an all-caps line's evidence for {@code language} is more than acronym-shaped tokens.+ *+ * <p>A word of {@link #SHOUTED_MIN_WORD} letters is required in every case, and then either+ * non-English diacritics or a matched stopword of {@link #SHOUTED_MIN_STOPWORD} letters.+ * Shouting is not a foreign language: {@code MON}, {@code LA}, {@code EST}, {@code COM} and+ * {@code DES} are hockey teams, states, time zones and radio bands that collide with French+ * stopwords, and nothing but their length tells them apart from words.+ *+ * @param tokens the word-scan tokens of the line+ * @param language the language the Latin guess named+ * @param diacriticRate the rate measured on the line+ */+ static boolean shoutedEvidence(List<String> tokens, String language, double diacriticRate) {+ boolean hasWord = false;+ for (String token : tokens) {+ if (codePointLength(token) >= SHOUTED_MIN_WORD) {+ hasWord = true;+ break;+ }+ }+ if (!hasWord) {+ return false;+ }+ if (diacriticRate >= LanguageTables.NON_EN_DIACRITIC_RATE) {+ return true;+ }+ Set<String> stop = LanguageTables.STOP_WORDS.get(language);+ if (stop == null) {+ // The reference's `_STOP.get(lang, set())` on a language it holds no list for: the+ // intersection is empty, so no stopword can clear the bar.+ return false;+ }+ for (String token : tokens) {+ String lowered = pythonLower(token);+ if (codePointLength(lowered) >= SHOUTED_MIN_STOPWORD && stop.contains(lowered)) {+ return true;+ }+ }+ return false;+ }+/*** Language code for one non-code line, or null when it does not name a foreign language.** <p>Same evidence bar as the segment scan: four words, a language the Latin guess will name,* and two <em>different</em> words of that language. Acronyms and slash compounds are not- * words.+ * words. A segment in all capitals keeps its acronym-shaped tokens -- shouting is not a+ * foreign language -- so it must also clear {@link #shoutedEvidence}.*/static String namedProseLanguage(String segment) {if (UnicodeTables.isBlank(segment) || hasCodeLine(segment)) {@@ -691,17 +853,22 @@ public final class LanguageDetection {}}String prose = String.join(" ", kept);- if (UnicodeTables.hasLower(prose)) {+ boolean allCaps = shouted(prose);+ if (!allCaps) {prose = blankUpperRuns(prose);}List<String> tokens = words(prose);if (tokens.size() < 4) {return null;}- String language = latinProfile(prose).language();+ LatinProfile profile = latinProfile(prose);+ String language = profile.language();if (language == null || "en".equals(language)) {return null;}+ if (allCaps && !shoutedEvidence(tokens, language, profile.diacriticRate())) {+ return null;+ }Set<String> stop = LanguageTables.STOP_WORDS.get(language);if (stop == null) {return null;@@ -763,9 +930,7 @@ public final class LanguageDetection {: Rounding.round4(1.0 - profile.getOrDefault("latin", 0.0));int nonLatinLetters = roundHalfEven(nonLatin * countAlpha(text));if ("latin".equals(script) && !nonLatinWords(text).isEmpty()- && (nonLatin >= LanguageTables.NON_LATIN_FRACTION- || (nonLatin >= LanguageTables.NON_LATIN_MIN_FRACTION- && nonLatinLetters >= LanguageTables.NON_LATIN_MIN_LETTERS))) {+ && nonLatinCarries(nonLatin, nonLatinLetters)) {// Non-Latin text is not for the English checkpoint even when Latin letters are the// plurality: a brand name or order code outvotes the CJK request around it letter for// letter, though one CJK character carries far more than a letter.@@ -778,14 +943,77 @@ public final class LanguageDetection {return new Analysis(script, profile, null, false, true, 0.0, nonLatin, null);}LatinProfile latin = latinProfile(text);+ String language = latin.language();+ // The acronym bar belongs here too, not only in the segment scan. The scan runs only once+ // a state already reads English overall, so a state that is *nothing but* an acronym line+ // -- or one diluted by fewer English lines than it takes to tip this verdict -- never+ // reached it and was named foreign outright: "MON DES EST LA" routed multilingual on its+ // own, and so did the same line under one or two lines of English. Vetoing here leaves+ // the text undecided, so looksNonEnglish still decides it: a shouted line with+ // non-English diacritics is kept.+ //+ // Neither bar applies when the state is carried by a caseless script. Both of them answer+ // the verdict by taking evidence *away* from the Latin letters, and looksNonEnglish, which+ // decides what is left, reads Latin diacritics only -- it cannot see the caseless half.+ // Nothing else catches the fall, either: the script promotion above needs nonLatinWords,+ // which drops any run starting with a capital, so a shouted mixed-script line has no+ // promotable word by construction. "PRIVET MON DES EST LA" written in Cyrillic is 35%+ // Cyrillic letters and was being sent to the English checkpoint on the strength of+ // disbelieving "MON DES EST LA". Whatever the Latin part is worth, text this far from+ // Latin is not English, so there is nothing for either bar to buy here and a wrong+ // language name costs nothing: the checkpoint is chosen on is_english.+ if (language != null && !"en".equals(language)+ && !nonLatinCarries(nonLatin, nonLatinLetters)) {+ if (shouted(text)) {+ if (!shoutedEvidence(words(text), language, latin.diacriticRate())) {+ language = null;+ }+ } else {+ // Mixed-case text: the segment scan has always held that an acronym is not a+ // word, but the whole-state verdict never applied that rule, so the acronyms+ // voted in it. That is what let a short English state be outvoted -- one or two+ // lines of English above "MON DES EST LA" still read as French overall, and only+ // at three did English win the margin. Re-take the verdict without the all-caps+ // runs. looksNonEnglish and the reported diacritic rate stay measured on the+ // original text, so a foreign word that happens to be shouted cannot be blanked+ // out of the diacritic safety net.+ //+ // Only *acronym-shaped* runs are blanked. Blanking every all-caps run deleted+ // emphasis capitals, which are ordinary in real prose and are usually on the+ // words that carry the sentence, so when the shouted words were the ones that+ // named the language the evidence simply went: "sag mir das HEUTIGE DATUM" and+ // "quiero cancelar mi PEDIDO POR FAVOR" both lost their verdict and routed+ // english. It also broke monotonicity -- the fully upper-cased form held the+ // shouted bar, but adding one lowercase English token moved the text to this+ // branch, where it had no bar at all. So a run is blanked only while every one of+ // them is short enough to be an acronym, by shoutedEvidence's own first test: a+ // run of SHOUTED_MIN_WORD letters is a word, and a word is not blanked out of its+ // own sentence. Measured over the 20,708 Latin-locale MASSIVE test rows,+ // emphasis-capitalised two ways: regressions 1,753 -> 293 and 3,835 -> 752.+ List<String> caps = upperLetterRuns(text);+ boolean allAcronyms = !caps.isEmpty();+ for (String run : caps) {+ if (codePointLength(run) >= SHOUTED_MIN_WORD) {+ allAcronyms = false;+ break;+ }+ }+ if (allAcronyms) {+ String blanked = blankUpperRuns(text);+ if (!blanked.equals(text)) {+ language = latinProfile(blanked).language();+ }+ }+ }+ }// Undecided is not English. Treating it as English sent every Latin-script language we// hold no stopwords for to the checkpoint that cannot read it, silently. When nothing// identifies the language, non-English letters or a shared Swedish-Danish marker can// still prefer the multilingual checkpoint; text with neither signal still goes to the// English one.- boolean undecided = latin.language() == null;- boolean english = "en".equals(latin.language()) || (undecided && !latin.looksNonEnglish());- return new Analysis("latin", profile, latin.language(), english, undecided,+ boolean undecided = language == null;+ boolean english = "en".equals(language) || (undecided && !latin.looksNonEnglish());+ return new Analysis("latin", profile, language, english, undecided,Rounding.round4(latin.diacriticRate()), nonLatin, null);}@@ -846,20 +1074,33 @@ public final class LanguageDetection {if (detected.english()) {continue;}+ // A named language still has to survive the prose scan: acronyms and slash compounds+ // are not words. But a veto is not a verdict of English. Dropping the line here+ // skipped the diacritic branch below, which exists for exactly this -- text that+ // carries non-English letters and that no stopword list can name. An upper-cased+ // German question with umlauts was vetoed for having no long stopword and then thrown+ // away, so a German field routed english; now it falls through and its umlauts carry+ // it. That is also why languageUndecided is no longer required below: a language named+ // and then disbelieved is in the same evidential position as one never named.+ //+ // The ASCII spelling of that same question is a different case and is not fixed here.+ // It has no diacritics to fall through to, and its longest matched German stopword is+ // three letters, so SHOUTED_MIN_STOPWORD vetoes it and nothing recovers it: genuine+ // German on the English checkpoint, one of the upper-cased casualties the reference+ // counts and not an example of what this branch buys.String language = detected.language();- if (language != null && !"en".equals(language)) {- if (namedProseLanguage(sample) == null) {+ boolean named = language != null && !"en".equals(language)+ && namedProseLanguage(sample) != null;+ if (!named) {+ if (!"latin".equals(detected.script()) && !"unknown".equals(detected.script())) {+ if (nonLatinWords(sample).isEmpty()+ || countAlpha(sample) < LanguageTables.NON_LATIN_MIN_LETTERS) {+ continue;+ }+ } else if (!(detected.diacriticRate() >= LanguageTables.NON_EN_DIACRITIC_RATE+ && words(sample).size() >= 4)) {continue;}- } else if (!"latin".equals(detected.script()) && !"unknown".equals(detected.script())) {- if (nonLatinWords(sample).isEmpty()- || countAlpha(sample) < LanguageTables.NON_LATIN_MIN_LETTERS) {- continue;- }- } else if (!(detected.languageUndecided()- && detected.diacriticRate() >= LanguageTables.NON_EN_DIACRITIC_RATE- && words(sample).size() >= 4)) {- continue;}int letters = countAlpha(sample);if (letters > bestLetters) {diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/ShoutedAcronymBarTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/ShoutedAcronymBarTest.javanew file mode 100644index 00000000..81f999a5--- /dev/null+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/lang/ShoutedAcronymBarTest.java@@ -0,0 +1,255 @@+package com.convaiinnovations.laya.lang;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;++/**+ * The caps bars, asserted directly rather than through the recorded fixture.+ *+ * <p>{@link LanguageDetectionParityTest} already covers every case in {@code lang_detect.json},+ * but it covers them as data: a rule can be removed and the fixture re-recorded from a Python+ * reference that lost the same rule, and nothing here would notice. These assertions name the+ * behaviour instead, so the rule has to be deleted from two places to disappear quietly.+ *+ * <p>The three cases the rule exists to balance are all here. An all-caps line of English+ * abbreviations that collide with French stopwords must not be named French; emphasis capitals in+ * genuine prose must not be blanked out of their own sentence; and a state carried by a caseless+ * script must be held to neither bar, because both work by discarding Latin evidence.+ */+class ShoutedAcronymBarTest {++ /** English abbreviations -- hockey teams, states, time zones, radio bands -- that read as French. */+ private static final String ACRONYMS = "MON LA EST COM DES";++ /** A genuinely Portuguese line, written without its accents as a customer would type it. */+ private static final String PORTUGUESE =+ "Nao consigo entrar na minha conta e a senha nao funciona de jeito nenhum";++ /**+ * The same line shouted, spelled out rather than upper-cased.+ *+ * <p>{@code String.toUpperCase()} without a locale is locale-sensitive -- under a Turkish+ * default it maps {@code i} to U+0130 -- and this line has four of them, so computing it+ * would make the assertion depend on the machine.+ */+ private static final String PORTUGUESE_SHOUTED =+ "NAO CONSIGO ENTRAR NA MINHA CONTA E A SENHA NAO FUNCIONA DE JEITO NENHUM";++ private static final String ENGLISH_OPENER =+ "The customer opened this ticket yesterday and we have asked for a screenshot.";++ private static final String ENGLISH_CLOSER =+ "We will escalate this to the platform team if it is not resolved today.";++ @Test+ @DisplayName("an all-caps line of English abbreviations does not name French")+ void acronymLineIsNotProse() {+ assertNull(LanguageDetection.namedProseLanguage(ACRONYMS),+ "bare abbreviations are acronym-shaped and name no language");+ assertEquals("pt", LanguageDetection.namedProseLanguage(PORTUGUESE),+ "the Portuguese line is still named, so the veto is not simply vetoing everything");+ }++ @Test+ @DisplayName("the genuinely foreign line decides a state the acronym line used to win")+ void foreignLineOutvotesTheAcronymLine() {+ String state = ENGLISH_OPENER + "\n" + ACRONYMS + "\n" + PORTUGUESE + "\n" + ENGLISH_CLOSER;+ LanguageDetection.Analysis analysis = LanguageDetection.analyse(state);+ assertEquals("pt", analysis.language(), "the Portuguese line decides, not the acronyms");+ assertFalse(analysis.english(), "a Portuguese message is not for the English checkpoint");+ assertFalse(analysis.languageUndecided(), "the language is named");+ assertEquals(PORTUGUESE, analysis.mixedSegment(), "and the line that named it is reported");+ }++ @Test+ @DisplayName("a state that is nothing but an acronym line is undecided, not French")+ void acronymOnlyStateIsUndecided() {+ LanguageDetection.Analysis analysis = LanguageDetection.analyse(ACRONYMS);+ assertNull(analysis.language(), "the whole-state verdict applies the bar too");+ assertTrue(analysis.languageUndecided(), "vetoing leaves the text undecided");+ // Undecided with no non-English letter is English: the veto takes the French verdict+ // away, it does not replace it with one of its own.+ assertTrue(analysis.english(), "nothing here is non-English, so the English checkpoint reads it");+ }++ @Test+ @DisplayName("one English line above the acronyms no longer routes multilingual")+ void shortEnglishStateIsNotOutvoted() {+ LanguageDetection.Analysis analysis =+ LanguageDetection.analyse("We have asked for a screenshot.\n" + ACRONYMS);+ assertEquals("en", analysis.language(), "the acronyms no longer vote in the verdict");+ assertTrue(analysis.english());+ }++ @Test+ @DisplayName("emphasis capitals keep the verdict they name")+ void emphasisCapitalsAreNotBlanked() {+ // Only acronym-shaped runs are blanked. Blanking every all-caps run deleted the words that+ // carried these two sentences, and both of them routed english with no language at all.+ LanguageDetection.Analysis german = LanguageDetection.analyse("sag mir das HEUTIGE DATUM");+ assertEquals("de", german.language(), "HEUTIGE and DATUM are words, not acronyms");+ assertFalse(german.english());++ LanguageDetection.Analysis spanish =+ LanguageDetection.analyse("quiero cancelar mi PEDIDO POR FAVOR");+ assertEquals("es", spanish.language(), "PEDIDO is a word, so no run is blanked");+ assertFalse(spanish.english());+ }++ @Test+ @DisplayName("an all-caps line with non-English letters clears the bar on its diacritics")+ void diacriticsStandInForALongStopword() {+ // Its longest matched German stopword is three letters, so the stopword half of the bar+ // vetoes it; the umlauts carry it, and the veto leaves it undecided rather than English.+ LanguageDetection.Analysis analysis = LanguageDetection.analyse("WIE SPÄT IST ES IN KÖLN");+ assertNull(analysis.language(), "no stopword list can name it");+ assertTrue(analysis.languageUndecided());+ assertFalse(analysis.english(), "a veto is not a verdict of English");+ }++ /**+ * A German field the whole-state verdict names but the prose scan disbelieves.+ *+ * <p>{@code wie} is the only German stopword it matches, and the prose scan wants two+ * <em>different</em> ones, so it is vetoed -- while the whole-state verdict, which has no such+ * rule, names it {@code de}. That disagreement is the only thing the restructure below+ * changes, so a field that merely clears both bars, or fails both, pins nothing.+ */+ private static final String VETOED_GERMAN = "wie Überprüfung wie Anmeldung";++ private static final String LONG_ENGLISH_NOTE =+ "The agent replied to the customer and closed the ticket today, and we have also "+ + "asked the platform team to confirm that the account is in good standing before "+ + "we escalate.";++ @Test+ @DisplayName("a vetoed field still reaches the multilingual checkpoint on its diacritics")+ void vetoIsNotAVerdictOfEnglishInAField() {+ Map<String, Object> state = new LinkedHashMap<>();+ state.put("note", LONG_ENGLISH_NOTE);+ state.put("msg", VETOED_GERMAN);++ // Three anchors, so a change that makes the assertion below pass for some other reason is+ // not mistaken for this rule holding. The joined window has to read English -- the note is+ // long enough to dilute the umlauts below the diacritic floor -- the prose scan has to+ // veto the field, and the whole-state verdict has to name it anyway. Only then is the+ // per-field pass the last thing that can catch it, which is the position the fix is about.+ assertTrue(LanguageDetection.analyseText(LanguageDetection.stateText(state)).english(),+ "the joined window has to read English, or this case tests nothing");+ assertNull(LanguageDetection.namedProseLanguage(VETOED_GERMAN),+ "the prose scan has to veto the field, or this case tests nothing");+ assertEquals("de", LanguageDetection.analyseText(VETOED_GERMAN).language(),+ "and the whole-state verdict has to name it, or this case tests nothing");+ assertNull(LanguageDetection.nonEnglishSegment(state, LanguageDetection.MAX_CHARS),+ "so the segment scan finds nothing");++ // Before the fix the field was dropped here: a named-then-disbelieved language took the+ // first branch and never reached the diacritic branch, which exists for exactly this.+ LanguageDetection.Analysis leaf = LanguageDetection.leafNonEnglish(VETOED_GERMAN);+ assertNotNull(leaf, "a veto is not a verdict of English");+ assertFalse(leaf.english());++ LanguageDetection.Analysis analysis = LanguageDetection.analyse(state);+ assertFalse(analysis.english(),+ "so the German field pulls the state off the English checkpoint");+ assertEquals("de", analysis.language());+ }++ @Test+ @DisplayName("a shouted field with umlauts is carried by the diacritic branch")+ void shoutedFieldFallsThroughOnItsDiacritics() {+ // The same position reached the other way: here the whole-state verdict vetoes the field+ // too, on the shouted bar, so it arrives undecided rather than named.+ Map<String, Object> state = new LinkedHashMap<>();+ state.put("note", LONG_ENGLISH_NOTE);+ state.put("msg", "WIE SPÄT IST ES IN KÖLN");+ assertTrue(LanguageDetection.analyseText(LanguageDetection.stateText(state)).english(),+ "the joined window has to read English, or this case tests nothing");+ assertFalse(LanguageDetection.analyse(state).english());+ assertTrue(LanguageDetection.analyse(state).languageUndecided());+ }++ @Test+ @DisplayName("a caseless script is held to neither caps bar")+ void caselessScriptKeepsTheBarsOff() {+ // 35% Cyrillic letters. Both bars answer the verdict by taking evidence away from the+ // Latin letters, and what decides what is left reads Latin diacritics only -- it cannot+ // see the caseless half. Text this far from Latin is not English whatever the Latin part+ // is worth, so the language name is allowed to stand.+ LanguageDetection.Analysis analysis =+ LanguageDetection.analyse("ПРИВЕТ MON DES EST LA");+ assertEquals("fr", analysis.language(), "the bar is kept off a mostly-non-Latin state");+ assertFalse(analysis.english(), "which is the only thing that decides the checkpoint");+ }++ @Test+ @DisplayName("shouted needs an uppercase character, so caseless text is not shouted")+ void shoutedNeedsAnUppercaseCharacter() {+ assertTrue(LanguageDetection.shouted(ACRONYMS));+ assertFalse(LanguageDetection.shouted("sag mir das HEUTIGE DATUM"), "one lowercase letter is enough");+ assertFalse(LanguageDetection.shouted("hola"));+ assertFalse(LanguageDetection.shouted("1234"), "no cased character at all");+ // Devanagari has no lowercase either; testing only for the absence of lowercase would call+ // this shouted and hold it to a bar its tokens cannot clear.+ assertFalse(LanguageDetection.shouted("हिन्दी बोलता"));+ }++ @Test+ @DisplayName("shouted evidence needs a long word, plus diacritics or a long stopword")+ void shoutedEvidenceNeedsMoreThanAcronyms() {+ assertFalse(LanguageDetection.shoutedEvidence(+ LanguageDetection.words(ACRONYMS), "fr", 0.0),+ "every token is acronym-shaped");+ assertTrue(LanguageDetection.shoutedEvidence(+ LanguageDetection.words(PORTUGUESE_SHOUTED), "pt", 0.0),+ "shouted Portuguese is still Portuguese");+ assertFalse(LanguageDetection.shoutedEvidence(List.of("PARA", "NAO"), "pt", 0.0),+ "no token reaches the word bar");+ assertTrue(LanguageDetection.shoutedEvidence(List.of("KOELN", "IST"), "de", 0.05),+ "non-English letters stand in for a long stopword");+ assertFalse(LanguageDetection.shoutedEvidence(List.of("SENHA", "NAO"), "zz", 0.0),+ "a language with no stopword list can match nothing");+ }++ @Test+ @DisplayName("both bars are at the value the reference swept to, not one letter lower")+ void barsSitWhereTheSweepPutThem() {+ // The acronym line above never reaches the stopword bar -- no token of it is long enough+ // to clear the word bar first -- so without these two the bars could each be loosened by+ // a letter with nothing failing. Each input clears one bar and is stopped by the other.++ // PARA and ESTA are four-letter Spanish stopwords: the stopword bar is cleared, and the+ // word bar is what stops it. At four it would pass.+ assertFalse(LanguageDetection.shoutedEvidence(+ LanguageDetection.words("PARA ESTA PARA ESTO"), "es", 0.0),+ "a four-letter stopword is not a five-letter word");++ // SPAET and KOELN clear the word bar, and the longest German stopword matched is `wie`,+ // three letters: the stopword bar is what stops it. At three it would pass.+ assertFalse(LanguageDetection.shoutedEvidence(+ LanguageDetection.words("WIE SPAET IST ES IN KOELN"), "de", 0.0),+ "three letters of stopword is not evidence");+ }++ @Test+ @DisplayName("the ASCII spelling of that German question is a casualty, and stays one")+ void asciiGermanIsAKnownCasualty() {+ // Not an endorsement: the reference counts this among the roughly 3,727 upper-cased rows+ // the pair of bars gives up, and says plainly that nothing recovers it -- it has no+ // diacritics to fall through to and no stopword long enough. It is asserted because it is+ // the cheapest behavioural pin on the stopword bar: lower the bar to three and genuine+ // German starts routing multilingual here, which is better, but it is not what the merged+ // reference does, and the Java port is not the place to change the rule.+ assertTrue(LanguageDetection.analyse("WIE SPAET IST ES IN KOELN").english(),+ "the bar vetoes it and no diacritic carries it");+ }+}