Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Made Cache compression level and lock timeout user configurable
luccioman28b451a0b375
diff --git a/defaults/yacy.init b/defaults/yacy.initindex f62f0971e..a9dc0abfc 100644--- a/defaults/yacy.init+++ b/defaults/yacy.init@@ -254,6 +254,14 @@ proxyCache = DATA/HTCACHE# default: 4 GigabyteproxyCacheSize = 4096+# The compression level for cached content+# Supported values ranging from 0 - no compression (lower CPU, higher disk usage), to 9 - best compression (higher CPU, lower disk use)+proxyCache.compressionLevel = 9++# Timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations+# When timeout occurs, loader should fall back to regular remote resource loading+proxyCache.sync.lockTimeout = 2000+# you can use the proxy with fresh/stale rules or in a always-fresh modeproxyAlwaysFresh = falsediff --git a/htroot/ConfigHTCache_p.html b/htroot/ConfigHTCache_p.htmlindex 2354a52f9..48bc2eb9a 100644--- a/htroot/ConfigHTCache_p.html+++ b/htroot/ConfigHTCache_p.html@@ -1,5 +1,5 @@-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">-<html xmlns="http://www.w3.org/1999/xhtml">+<!DOCTYPE html>+<html lang="en"><head><title>YaCy '#[clientname]#': Hypertext Cache Configuration</title>#%env/templates/metas.template%#@@ -18,10 +18,24 @@<dl><dt><label for="HTCachePath">The path where the cache is stored</label></dt><dd><input name="HTCachePath" id="HTCachePath" type="text" size="20" maxlength="300" value="#[HTCachePath]#" /></dd>- <dt><label for="actualCacheSize">The current size of the cache</label></dt>+ <dt><label>The current size of the cache</label></dt><dd><span id="actualCacheSize">#[actualCacheSize]# MB for #[actualCacheDocCount]# files, #[docSizeAverage]# KB / file in average </span></dd><dt><label for="maxCacheSize">The maximum size of the cache</label></dt><dd><input name="maxCacheSize" id="maxCacheSize" type="text" size="8" maxlength="24" value="#[maxCacheSize]#" /> MB</dd>+ <dt><label for="compressionLevel">Compression level</label></dt>+ <dd><select id="compressionLevel" name="compressionLevel">+ #{compressionLevels}#+ <option value="#[value]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>+ #{/compressionLevels}#+ </select></dd>+ <dt><label for="lockTimeout" aria-describedby="timeoutInfo">Concurrent access timeout</label>+ <span class="info">+ <img src="env/grafics/i16.gif" width="16" height="16" alt="Concurrent access timeout info"/>+ <span class="infobox" id="timeoutInfo">The maximum time to wait for acquiring a synchronization lock on concurrent get/store cache operations.+ Beyond this limit, the crawler or proxy falls back to regular remote resource loading.</span>+ </span>+ </dt>+ <dd><input name="lockTimeout" id="lockTimeout" type="number" min="10" max="600000" value="#[lockTimeout]#"/> milliseconds</dd><dt> </dt><dd><input type="submit" name="set" value="Set" class="btn btn-primary"/></dd></dl>diff --git a/htroot/ConfigHTCache_p.java b/htroot/ConfigHTCache_p.javaindex 6005e1d2a..88baf9c95 100644--- a/htroot/ConfigHTCache_p.java+++ b/htroot/ConfigHTCache_p.java@@ -28,6 +28,7 @@import java.io.File;import java.io.IOException;+import java.util.zip.Deflater;import net.yacy.cora.protocol.RequestHeader;import net.yacy.crawler.data.Cache;@@ -61,6 +62,19 @@ public class ConfigHTCache_p {final int newProxyCacheSize = Math.max(post.getInt("maxCacheSize", 64), 0);env.setConfig(SwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize);Cache.setMaxCacheSize(newProxyCacheSize * 1024L * 1024L);++ /* Compression level*/+ /* Ensure a value within the range supported by the Deflater class */+ final int newCompressionLevel = Math.max(Deflater.NO_COMPRESSION, Math.min(Deflater.BEST_COMPRESSION,+ post.getInt("compressionLevel", SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT)));+ env.setConfig(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL, newCompressionLevel);+ Cache.setCompressionLevel(newCompressionLevel);++ /* Synchronization lock timeout */+ final long newLockTimeout = Math.max(10, Math.min(60000,+ post.getLong("lockTimeout", SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT)));+ env.setConfig(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT, newLockTimeout);+ Cache.setLockTimeout(newLockTimeout);}if (post != null && post.containsKey("deletecomplete")) {@@ -73,6 +87,32 @@ public class ConfigHTCache_p {}prop.put("HTCachePath", env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT));++ /* Compression levels */+ final int configuredCompressionLevel = env.getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL,+ SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT);+ int levelsCount = 0;+ for(int level = Deflater.NO_COMPRESSION; level <= Deflater.BEST_COMPRESSION; level++) {+ if(level == configuredCompressionLevel) {+ prop.put("compressionLevels_" + levelsCount + "_selected", "1");+ } else {+ prop.put("compressionLevels_" + levelsCount + "_selected", "0");+ }+ prop.put("compressionLevels_" + levelsCount + "_value", level);+ prop.put("compressionLevels_" + levelsCount + "_name", level);+ if(level == Deflater.NO_COMPRESSION) {+ prop.put("compressionLevels_" + levelsCount + "_name", "0 - No compression");+ } else if(level == Deflater.BEST_SPEED) {+ prop.put("compressionLevels_" + levelsCount + "_name", Deflater.BEST_SPEED + " - Best speed");+ } else if(level == Deflater.BEST_COMPRESSION) {+ prop.put("compressionLevels_" + levelsCount + "_name", Deflater.BEST_COMPRESSION + " - Best compression");+ }+ levelsCount++;+ }+ prop.put("compressionLevels", levelsCount);++ prop.put("lockTimeout", env.getConfigLong(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT,+ SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT));prop.put("actualCacheSize", Cache.getActualCacheSize() / 1024 / 1024);prop.put("actualCacheDocCount", Cache.getActualCacheDocCount());prop.put("docSizeAverage", Cache.getActualCacheDocCount() == 0 ? 0 : Cache.getActualCacheSize() / Cache.getActualCacheDocCount() / 1024);diff --git a/source/net/yacy/crawler/data/Cache.java b/source/net/yacy/crawler/data/Cache.javaindex 96a97872f..bc5189e4f 100644--- a/source/net/yacy/crawler/data/Cache.java+++ b/source/net/yacy/crawler/data/Cache.java@@ -84,8 +84,9 @@ public final class Cache {* @param peerSalt peer identifier* @param cacheSizeMax maximum cache size in bytes* @param lockTimeout maximum time (in milliseconds) to acquire a synchronization lock on store() and getContent()+ * @param compressionLevel the compression level : supported values ranging from 0 - no compression, to 9 - best compression*/- public static void init(final File htCachePath, final String peerSalt, final long cacheSizeMax, final long lockTimeout) {+ public static void init(final File htCachePath, final String peerSalt, final long cacheSizeMax, final long lockTimeout, final int compressionLevel) {cachePath = htCachePath;maxCacheSize = cacheSizeMax;@@ -116,7 +117,7 @@ public final class Cache {try {fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, DEFAULT_BACKEND_BUFFER_SIZE, false, true);fileDBunbuffered.setMaxSize(maxCacheSize);- fileDB = new Compressor(fileDBunbuffered, DEFAULT_COMPRESSOR_BUFFER_SIZE, lockTimeout);+ fileDB = new Compressor(fileDBunbuffered, DEFAULT_COMPRESSOR_BUFFER_SIZE, lockTimeout, compressionLevel);} catch (final IOException e) {ConcurrentLog.logException(e);// try a healing@@ -125,7 +126,7 @@ public final class Cache {try {fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, DEFAULT_BACKEND_BUFFER_SIZE, false, true);fileDBunbuffered.setMaxSize(maxCacheSize);- fileDB = new Compressor(fileDBunbuffered, DEFAULT_COMPRESSOR_BUFFER_SIZE, lockTimeout);+ fileDB = new Compressor(fileDBunbuffered, DEFAULT_COMPRESSOR_BUFFER_SIZE, lockTimeout, compressionLevel);} catch (final IOException ee) {ConcurrentLog.logException(e);}@@ -226,6 +227,22 @@ public final class Cache {public static long getActualCacheDocCount() {return fileDBunbuffered.size();}++ /**+ * Set the new content compression level+ * @param newCompressionLevel the new compression level. Supported values between 0 (no compression) and 9 (best compression)+ */+ public static void setCompressionLevel(final int newCompressionLevel) {+ fileDB.setCompressionLevel(newCompressionLevel);+ }++ /**+ * Set the new synchronization lock timeout.+ * @param lockTimeout the new synchronization lock timeout (in milliseconds).+ */+ public static void setLockTimeout(final long lockTimeout) {+ fileDB.setLockTimeout(lockTimeout);+ }/*** close the databasesdiff --git a/source/net/yacy/kelondro/blob/Compressor.java b/source/net/yacy/kelondro/blob/Compressor.javaindex 1bbb2c233..a78cb3cd1 100644--- a/source/net/yacy/kelondro/blob/Compressor.java+++ b/source/net/yacy/kelondro/blob/Compressor.java@@ -61,19 +61,32 @@ public class Compressor implements BLOB, Iterable<byte[]> {/** Total size (in bytes) of uncompressed entries in buffer */private long bufferlength;++ /** Maximum {@link #bufferlength} value before compressing and flushing to the backend */private final long maxbufferlength;/** Maximum time (in milliseconds) to acquire a synchronization lock on get() and insert() */- private final long lockTimeout;+ private long lockTimeout;/** Synchronization lock */private final ReentrantLock lock;-- public Compressor(final BLOB backend, final long buffersize, final long lockTimeout) {++ /** The compression level */+ private int compressionLevel;++ /**+ * @param backend the backend storage+ * @param buffersize the maximum total size (in bytes) of uncompressed in-memory entries before compressing and flushing to the backend+ * @param lockTimeout maximum time to acquire a synchronization lock on get() and insert() operations+ * @param compressionLevel the compression level : supported values ranging from 0 - no compression, to 9 - best compression+ */+ public Compressor(final BLOB backend, final long buffersize, final long lockTimeout, final int compressionLevel) {this.backend = backend;this.maxbufferlength = buffersize;this.lockTimeout = lockTimeout;this.lock = new ReentrantLock();+ /* Ensure a value within the range supported by the Deflater class */+ this.compressionLevel = Math.max(Deflater.NO_COMPRESSION, Math.min(Deflater.BEST_COMPRESSION, compressionLevel));initBuffer();}@@ -125,21 +138,21 @@ public class Compressor implements BLOB, Iterable<byte[]> {}}- private static byte[] compress(final byte[] b) {+ private static byte[] compress(final byte[] b, final int compressionLevel) {final int l = b.length;if (l < 100) return markWithPlainMagic(b);- final byte[] bb = compressAddMagic(b);+ final byte[] bb = compressAddMagic(b, compressionLevel);if (bb.length >= l) return markWithPlainMagic(b);return bb;}- private static byte[] compressAddMagic(final byte[] b) {+ private static byte[] compressAddMagic(final byte[] b, final int compressionLevel) {// compress a byte array and add a leading magic for the compressiontry {//System.out.print("/(" + cdr + ")"); // DEBUGfinal ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length / 5);baos.write(gzipMagic);- final OutputStream os = new GZIPOutputStream(baos, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};+ final OutputStream os = new GZIPOutputStream(baos, 65536){{def.setLevel(compressionLevel);}};os.write(b);os.close();baos.close();@@ -213,7 +226,7 @@ public class Compressor implements BLOB, Iterable<byte[]> {b = this.buffer.remove(key);if (b != null) {this.bufferlength = this.bufferlength - b.length;- this.backend.insert(key, compress(b));+ this.backend.insert(key, compress(b, this.compressionLevel));return b;}} finally {@@ -412,7 +425,7 @@ public class Compressor implements BLOB, Iterable<byte[]> {final Map.Entry<byte[], byte[]> entry = this.buffer.entrySet().iterator().next();this.buffer.remove(entry.getKey());try {- this.backend.insert(entry.getKey(), compress(entry.getValue()));+ this.backend.insert(entry.getKey(), compress(entry.getValue(), this.compressionLevel));this.bufferlength -= entry.getValue().length;return true;} catch (final IOException e) {@@ -457,6 +470,23 @@ public class Compressor implements BLOB, Iterable<byte[]> {insert(key, c);return reduction;}++ /**+ * Set the new content compression level.+ * @param compressionLevel the new compression level. Supported values between 0 (no compression) and 9 (best compression).+ */+ public void setCompressionLevel(final int compressionLevel) {+ /* Ensure a value within the range supported by the Deflater class */+ this.compressionLevel = Math.max(Deflater.NO_COMPRESSION, Math.min(Deflater.BEST_COMPRESSION, compressionLevel));+ }++ /**+ * Set the new synchronization lock timeout.+ * @param lockTimeout the new synchronization lock timeout (in milliseconds).+ */+ public void setLockTimeout(final long lockTimeout) {+ this.lockTimeout = lockTimeout;+ }}diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.javaindex 1ae05c993..cd7e4850e 100644--- a/source/net/yacy/search/Switchboard.java+++ b/source/net/yacy/search/Switchboard.java@@ -723,7 +723,11 @@ public final class Switchboard extends serverSwitch {this.log.info("HTCACHE Path = " + this.htCachePath.getAbsolutePath());final long maxCacheSize =1024L * 1024L * Long.parseLong(getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte- Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize, 2000);+ Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize,+ getConfigLong(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT,+ SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT),+ getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL,+ SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT));final File transactiondir = new File(this.htCachePath, "snapshots");Transactions.init(transactiondir);diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.javaindex ff69a6abe..098403711 100644--- a/source/net/yacy/search/SwitchboardConstants.java+++ b/source/net/yacy/search/SwitchboardConstants.java@@ -26,6 +26,8 @@package net.yacy.search;+import java.util.zip.Deflater;+import net.yacy.cora.order.Digest;import net.yacy.server.http.RobotsTxtConfig;@@ -390,6 +392,19 @@ public final class SwitchboardConstants {*/public static final String HTCACHE_PATH = "proxyCache";public static final String HTCACHE_PATH_DEFAULT = "DATA/HTCACHE";++ /** Key of the setting configuring the cache synchronization */+ public static final String HTCACHE_COMPRESSION_LEVEL = "proxyCache.compressionLevel";++ /** Default compression level for cached content */+ public static final int HTCACHE_COMPRESSION_LEVEL_DEFAULT = Deflater.BEST_COMPRESSION;++ /** Key of the setting configuring Cache synchronization lock timeout on getContent/store operations*/+ public static final String HTCACHE_SYNC_LOCK_TIMEOUT = "proxyCache.sync.lockTimeout";++ /** Default timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations */+ public static final long HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT = 2000;+public static final String RELEASE_PATH = "releases";public static final String RELEASE_PATH_DEFAULT = "DATA/RELEASE";diff --git a/test/java/net/yacy/crawler/data/CacheTest.java b/test/java/net/yacy/crawler/data/CacheTest.javaindex bb71b5e84..8c8fe5d96 100644--- a/test/java/net/yacy/crawler/data/CacheTest.java+++ b/test/java/net/yacy/crawler/data/CacheTest.java@@ -34,6 +34,7 @@ import java.util.List;import java.util.Random;import java.util.concurrent.TimeUnit;import java.util.logging.LogManager;+import java.util.zip.Deflater;import org.apache.http.HttpStatus;import org.junit.After;@@ -68,7 +69,7 @@ public class CacheTest {Cache.init(new File(System.getProperty("java.io.tmpdir") + File.separator + "testCache"), "peerSalt",Math.max(Math.max(TEXT_CONTENT.getBytes(StandardCharsets.UTF_8).length * 10,Cache.DEFAULT_COMPRESSOR_BUFFER_SIZE * 2), Cache.DEFAULT_BACKEND_BUFFER_SIZE * 2),- 2000);+ 2000, Deflater.BEST_COMPRESSION);Cache.clear();}@@ -497,9 +498,11 @@ public class CacheTest {final long sleepTime = 0;/* Maximum waiting time (in ms) for acquiring a synchronization lock */final long lockTimeout = 2000;+ /* The backend compression level */+ final int compressionLevel = Deflater.BEST_COMPRESSION;Cache.init(new File(System.getProperty("java.io.tmpdir") + File.separator + "yacyTestCache"), "peerSalt",- cacheMaxSize, lockTimeout);+ cacheMaxSize, lockTimeout, compressionLevel);Cache.clear();System.out.println("Cache initialized with a maximum size of " + cacheMaxSize + " bytes.");