Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Added gzip compression of responses returned to user-agents accepting it

luccioman387d646c0e5d
diff --git a/defaults/yacy.init b/defaults/yacy.init
index 6f2b4601b..6157e8051 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -92,6 +92,11 @@ server.maxTrackingCount = 1000
# maximum number of hosts that are tracked
server.maxTrackingHostCount = 100
+# When set to true, enable gzip compression of HTTP responses when the user-agent (browser, another YaCy peer or any kind of client)
+# accept gzip encoding (by including gzip in a 'Accept-Encoding' HTTP request header)
+# This adds some processing overhead, but reduces the amount of bytes sent over network.
+server.response.compress.gzip = true
+
# Global HTTP Referrer policy delivered by meta tag (see https://www.w3.org/TR/referrer-policy/ for available policies)
# Can be left empty : the browser should then fallback to the default "no-referrer-when-downgrade" policy
# Be careful, some policies will also affect YaCy internal links : "no-referrer", "same-origin", "origin" and "strict-origin". This can be useful
diff --git a/htroot/SettingsAck_p.html b/htroot/SettingsAck_p.html
index 04c5be21d..3c67481cc 100644
--- a/htroot/SettingsAck_p.html
+++ b/htroot/SettingsAck_p.html
@@ -208,6 +208,9 @@
<td>Shutdown port</td><td> <span class="settingsValue">#[port.shutdown]#</span></td>
</tr>
</table>
+ ::<!-- 37: Compression settings changed -->
+ <p>Compression settings have been saved.</p>
+
#(/info)#
<p></p>
#(needsRestart)#
diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java
index aecc0e4ad..ca31a6748 100644
--- a/htroot/SettingsAck_p.java
+++ b/htroot/SettingsAck_p.java
@@ -258,6 +258,23 @@ public class SettingsAck_p {
prop.putHTML("info_filter", filter);
return prop;
}
+
+ /* Compression settings */
+ if (post.containsKey("serverCompression")) {
+ // set backlink
+ prop.put("needsRestart_referer", "Settings_p.html?page=ServerAccess");
+
+ final boolean oldValue = env.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+ SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT);
+ final boolean newValue = post.containsKey(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP);
+ env.setConfig(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP, newValue);
+
+ prop.put("needsRestart", newValue != oldValue);
+
+ /* server compression settings saved */
+ prop.put("info", "37");
+ return prop;
+ }
if (post.containsKey("proxysettings")) {
// set backlink
diff --git a/htroot/Settings_ServerAccess.inc b/htroot/Settings_ServerAccess.inc
index e75ecf085..6a97978ed 100644
--- a/htroot/Settings_ServerAccess.inc
+++ b/htroot/Settings_ServerAccess.inc
@@ -66,4 +66,32 @@
</table>
</fieldset>
+<fieldset>
+ <legend>Compression settings</legend>
+
+ <div class="form-group">
+ <div class="col-sm-3">
+ <div class="checkbox">
+ <label>
+ <input name="server.response.compress.gzip" id="responseCompressGzip"
+ type="checkbox" #(server.response.compress.gzip)#::checked#(/server.response.compress.gzip)#
+ aria-describedby="responseCompressGzipInfo"/>
+ Compress responses with gzip
+ </label>
+ </div>
+ </div>
+ <div class="col-sm-9" id="responseCompressGzipInfo">
+ When checked (default), HTTP responses can be compressed using gzip.
+ The requesting user-agent (a web browser, another YaCy peer or any other tool) uses the header 'Accept-Encoding' to tell whether it accepts gzip compression or not.
+ This adds some processing overhead, but can significantly reduce the amount of bytes transmitted over the network.
+ </div>
+ </div>
+ <div class="form-group">
+ <div class="col-sm-12">
+ <input type="submit" class="btn btn-primary" name="serverCompression" value="Submit" aria-describedby="submitInfo"/>
+ <em id="submitInfo">Changes need a server restart.</em>
+ </div>
+ </div>
+</fieldset>
+
</form>
\ No newline at end of file
diff --git a/htroot/Settings_p.java b/htroot/Settings_p.java
index e2b6f2d2b..25b0fb837 100644
--- a/htroot/Settings_p.java
+++ b/htroot/Settings_p.java
@@ -135,6 +135,11 @@ public final class Settings_p {
// server access filter
prop.putHTML("serverfilter", env.getConfig("serverClient", "*"));
+ /* gzip compression of HTTP responses */
+ prop.put(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+ env.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+ SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT));
+
// server password
prop.put("serveruser","server");
diff --git a/source/net/yacy/http/Jetty9HttpServerImpl.java b/source/net/yacy/http/Jetty9HttpServerImpl.java
index f692fb27a..feb70e6e8 100644
--- a/source/net/yacy/http/Jetty9HttpServerImpl.java
+++ b/source/net/yacy/http/Jetty9HttpServerImpl.java
@@ -48,6 +48,7 @@ import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.InetAccessHandler;
+import org.eclipse.jetty.server.handler.gzip.GzipHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.resource.Resource;
import org.eclipse.jetty.util.ssl.SslContextFactory;
@@ -142,6 +143,19 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
sholder.setAsyncSupported(true); // needed for YaCyQoSFilter
//sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html
htrootContext.addServlet(sholder, "/*");
+
+ /* Handle gzip compression of responses to user agents accepting it */
+ final GzipHandler gzipHandler;
+ if (sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+ SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) {
+ gzipHandler = new GzipHandler();
+ /*
+ * Ensure decompression of requests body is disabled : it is already handled by
+ * the GZIPRequestWrapper in the YaCyDefaultServlet
+ */
+ gzipHandler.setInflateBufferSize(0);
+ htrootContext.setGzipHandler(gzipHandler);
+ }
// -----------------------------------------------------------------------------
// here we set and map the mandatory servlets, needed for typical YaCy operation
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index b5ae101ca..98792d9b2 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -484,6 +484,12 @@ public final class SwitchboardConstants {
/** Setting key of the property that collects the names of all servlets that have been used so far. */
public static final String SERVER_SERVLETS_CALLED = "server.servlets.called";
+
+ /** Key of the setting controlling whether HTTP responses should be compressed with gzip when the user-agent accepts it (by including gzip in a 'Accept-Encoding' HTTP request header) */
+ public static final String SERVER_RESPONSE_COMPRESS_GZIP = "server.response.compress.gzip";
+
+ /** Default setting value controlling whether HTTP responses should be compressed */
+ public static final boolean SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT = true;
/*