Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Improved synonyms memory footprint.
luccioman568e3dde6a51
diff --git a/source/net/yacy/cora/language/synonyms/SynonymLibrary.java b/source/net/yacy/cora/language/synonyms/SynonymLibrary.javaindex 4992a2571..82fa88095 100644--- a/source/net/yacy/cora/language/synonyms/SynonymLibrary.java+++ b/source/net/yacy/cora/language/synonyms/SynonymLibrary.java@@ -49,6 +49,9 @@ public class SynonymLibrary {lib.clear();if (!path.exists() || !path.isDirectory()) return;final String[] files = path.list();+ /* Global map of all known distinct words : thus enable reuse of the same word String instance+ * appearing multiple times in different synonyms sets */+ final Map<String, String> distinctWords = new HashMap<>();for (final String f: files) {File ff = new File(path, f);String line;@@ -62,12 +65,20 @@ public class SynonymLibrary {String[] words = CommonPattern.COMMA.split(line);Set<String> synonyms = new HashSet<String>();Set<String> keys = new HashSet<String>();- for (String s: words) {- s = s.trim();- if (s.length() < 2) continue;- String t = s.toLowerCase();- synonyms.add(t);- keys.add(t.substring(0, 2));+ for (String word: words) {+ word = word.trim();+ if (word.length() < 2) continue;+ String lowCaseWord = word.toLowerCase();+ String kownWord = distinctWords.get(lowCaseWord);+ if(kownWord != null) {+ /* This word is already known : let's use the existing String instance from the synonyms map to gain memory space */+ lowCaseWord = kownWord;+ } else {+ /* First encounter of this word : let's add it to the global map of known words */+ distinctWords.put(lowCaseWord, lowCaseWord);+ }+ synonyms.add(lowCaseWord);+ keys.add(lowCaseWord.substring(0, 2));}for (String key: keys) {List<Set<String>> symsetlist = lib.get(key);