Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Improved synonyms memory footprint.

luccioman568e3dde6a51
diff --git a/source/net/yacy/cora/language/synonyms/SynonymLibrary.java b/source/net/yacy/cora/language/synonyms/SynonymLibrary.java
index 4992a2571..82fa88095 100644
--- a/source/net/yacy/cora/language/synonyms/SynonymLibrary.java
+++ b/source/net/yacy/cora/language/synonyms/SynonymLibrary.java
@@ -49,6 +49,9 @@ public class SynonymLibrary {
lib.clear();
if (!path.exists() || !path.isDirectory()) return;
final String[] files = path.list();
+ /* Global map of all known distinct words : thus enable reuse of the same word String instance
+ * appearing multiple times in different synonyms sets */
+ final Map<String, String> distinctWords = new HashMap<>();
for (final String f: files) {
File ff = new File(path, f);
String line;
@@ -62,12 +65,20 @@ public class SynonymLibrary {
String[] words = CommonPattern.COMMA.split(line);
Set<String> synonyms = new HashSet<String>();
Set<String> keys = new HashSet<String>();
- for (String s: words) {
- s = s.trim();
- if (s.length() < 2) continue;
- String t = s.toLowerCase();
- synonyms.add(t);
- keys.add(t.substring(0, 2));
+ for (String word: words) {
+ word = word.trim();
+ if (word.length() < 2) continue;
+ String lowCaseWord = word.toLowerCase();
+ String kownWord = distinctWords.get(lowCaseWord);
+ if(kownWord != null) {
+ /* This word is already known : let's use the existing String instance from the synonyms map to gain memory space */
+ lowCaseWord = kownWord;
+ } else {
+ /* First encounter of this word : let's add it to the global map of known words */
+ distinctWords.put(lowCaseWord, lowCaseWord);
+ }
+ synonyms.add(lowCaseWord);
+ keys.add(lowCaseWord.substring(0, 2));
}
for (String key: keys) {
List<Set<String>> symsetlist = lib.get(key);