Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
adjust Tokenizer sentence count to ignore repeated punktuation (like !!!! ) + remove unused sentenceword map (we use only the count) + upd test case for sentence count
regerae3717d0870a
diff --git a/source/net/yacy/document/Tokenizer.java b/source/net/yacy/document/Tokenizer.javaindex 0420194f5..f730c8b43 100644--- a/source/net/yacy/document/Tokenizer.java+++ b/source/net/yacy/document/Tokenizer.java@@ -68,7 +68,6 @@ public class Tokenizer {this.words = new TreeMap<String, Word>(NaturalOrder.naturalComparator);this.synonyms = new LinkedHashSet<String>();assert text != null;- final Set<String> currsentwords = new HashSet<String>();String[] wordcache = new String[LibraryProvider.autotagging.getMaxWordsInTerm() - 1];for (int i = 0; i < wordcache.length; i++) wordcache[i] = "";String k;@@ -89,9 +88,9 @@ public class Tokenizer {// handle punktuation (start new sentence)if (word.length() == 1 && SentenceReader.punctuation(word.charAt(0))) {// store sentence- currsentwords.clear();+ if (wordInSentenceCounter > 1) // if no word in sentence repeated punktuation ".....", don't count as sentence+ allsentencecounter++;wordInSentenceCounter = 1;- allsentencecounter++;continue;}if (word.length() < wordminsize) continue;@@ -160,7 +159,6 @@ public class Tokenizer {// store wordallwordcounter++;- currsentwords.add(word);Word wsp = this.words.get(word);if (wsp != null) {// word already exists@@ -214,7 +212,7 @@ public class Tokenizer {// store resultthis.RESULT_NUMB_WORDS = allwordcounter;// if text doesn't end with punktuation but has words after last found sentence, inc sentence count for trailing text.- this.RESULT_NUMB_SENTENCES = allsentencecounter + (currsentwords.size() > 0 ? 1 : 0);+ this.RESULT_NUMB_SENTENCES = allsentencecounter + (wordInSentenceCounter > 1 ? 1 : 0);}public Map<String, Word> words() {diff --git a/test/java/net/yacy/document/TokenizerTest.java b/test/java/net/yacy/document/TokenizerTest.javaindex e54807105..8f5edd7c9 100644--- a/test/java/net/yacy/document/TokenizerTest.java+++ b/test/java/net/yacy/document/TokenizerTest.java@@ -2,7 +2,9 @@package net.yacy.document;import java.net.MalformedURLException;+import java.util.HashSet;import java.util.Map;+import java.util.Set;import net.yacy.cora.document.WordCache;import net.yacy.kelondro.data.word.Word;import org.junit.Test;@@ -36,4 +38,24 @@ public class TokenizerTest {assertEquals("occurence of 'words' ", 2, w.occurrences());}+ /**+ * Test of RESULT_NUMB_SENTENCES, of class Tokenizer.+ */+ @Test+ public void testNumberOfSentences() {+ Set<String> testText = new HashSet();+ // text with 5 sentences+ testText.add("Sentence One. Sentence Two. Comment on this. This is sentence four! Good By................");+ testText.add("Sentence One. Sentence two. Sentence 3? Sentence 4! Sentence w/o punktuation at end of text");+ testText.add("!!! ! ! ! Sentence One. Sentence two. Sentence 3? Sentence 4! Sentence 5 ! ! ! !!!");++ WordCache meaningLib = new WordCache(null);+ boolean doAutotagging = false;+ VocabularyScraper scraper = null;+ for (String text : testText) {+ Tokenizer t = new Tokenizer(null, text, meaningLib, doAutotagging, scraper);+ System.out.println(t.RESULT_NUMB_WORDS);+ assertEquals("Tokenizer.RESULT_NUMB_SENTENCES", 5, t.RESULT_NUMB_SENTENCES);+ }+ }}