Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
change unused servletProperties entry CONNECTION_PROP_CLIENT_REQUEST_HEADER to use directly HttpServletRequest. This is used to get the http protocol version in HTTPDProxyHandler.fulfillRequestFromWeb() for error response to client. - adjust YaCyProxyServlet and UrlProxyServlet accordingly - use more http_version constants in headerframework and httpdeamon - equalize servlets (3) use of HeaderFramework.CONNECTION_PROP_HOST to HeaderFramework.HOST
reger3b47a07dd1d2
diff --git a/htroot/QuickCrawlLink_p.java b/htroot/QuickCrawlLink_p.javaindex da3e81cd8..3e7158a44 100644--- a/htroot/QuickCrawlLink_p.java+++ b/htroot/QuickCrawlLink_p.java@@ -70,7 +70,7 @@ public class QuickCrawlLink_p {prop.put("mode", "0");// get the http host header- final String hostSocket = header.get(HeaderFramework.CONNECTION_PROP_HOST);+ final String hostSocket = header.get(HeaderFramework.HOST);//String host = hostSocket;int port = 80;diff --git a/htroot/YaCySearchPluginFF.java b/htroot/YaCySearchPluginFF.javaindex 199bf1b73..9ddd0c5eb 100644--- a/htroot/YaCySearchPluginFF.java+++ b/htroot/YaCySearchPluginFF.java@@ -47,7 +47,7 @@ public class YaCySearchPluginFF {final serverObjects prop = new serverObjects();// getting the http host header- final String hostSocket = header.get(HeaderFramework.CONNECTION_PROP_HOST);+ final String hostSocket = header.get(HeaderFramework.HOST);String host = hostSocket;int port = 80;diff --git a/htroot/autoconfig.java b/htroot/autoconfig.javaindex 4be45aab7..3ce7b313c 100644--- a/htroot/autoconfig.java+++ b/htroot/autoconfig.java@@ -52,7 +52,7 @@ public class autoconfig {final boolean yacyonly = env.getConfigBool(SwitchboardConstants.PROXY_YACY_ONLY, false);// get the http host header- final String hostSocket = header.get(HeaderFramework.CONNECTION_PROP_HOST);+ final String hostSocket = header.get(HeaderFramework.HOST);String host = hostSocket;int port = 80;diff --git a/source/net/yacy/cora/protocol/HeaderFramework.java b/source/net/yacy/cora/protocol/HeaderFramework.javaindex 5048577c8..c013b9470 100644--- a/source/net/yacy/cora/protocol/HeaderFramework.java+++ b/source/net/yacy/cora/protocol/HeaderFramework.java@@ -196,6 +196,9 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri}/* PROPERTIES: General properties */+ // Properties are used to internally store or remember header values and additional connection information+ // One of the usages is in proxy operation to prepare header values to be set as header values upon connection+ // * use of properties as header values is discouraged (e.g. as proxy transmits it as arbitrary headers) [2016-8-21]public static final String CONNECTION_PROP_HTTP_VER = "HTTP";public static final String CONNECTION_PROP_PROTOCOL = "PROTOCOL";public static final String CONNECTION_PROP_HOST = "HOST";@@ -207,16 +210,11 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stripublic static final String CONNECTION_PROP_ARGS = "ARGS";public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP";public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT";- //public static final String CONNECTION_PROP_KEEP_ALIVE_COUNT = "KEEP-ALIVE_COUNT";- //public static final String CONNECTION_PROP_REQUESTLINE = "REQUESTLINE";- //public static final String CONNECTION_PROP_PREV_REQUESTLINE = "PREVREQUESTLINE";public static final String CONNECTION_PROP_REQUEST_START = "REQUEST_START";public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END";- //public static final String CONNECTION_PROP_INPUTSTREAM = "INPUTSTREAM";- //public static final String CONNECTION_PROP_OUTPUTSTREAM = "OUTPUTSTREAM";/* PROPERTIES: Client -> Proxy */- public static final String CONNECTION_PROP_CLIENT_REQUEST_HEADER = "CLIENT_REQUEST_HEADER";+ public static final String CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST = "CLIENT_HTTPSERVLETREQUEST";/* PROPERTIES: Proxy -> Client */public static final String CONNECTION_PROP_PROXY_RESPOND_CODE = "PROXY_RESPOND_CODE";@@ -546,7 +544,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Striif (theHeader == null) throw new IllegalArgumentException();// setting the http version if it was not already set- if (httpVersion == null) httpVersion = "HTTP/1.0";+ if (httpVersion == null) httpVersion = HTTP_VERSION_1_0;// setting the status text if it was not already setif ((httpStatusText == null)||(httpStatusText.length()==0)) {diff --git a/source/net/yacy/http/servlets/UrlProxyServlet.java b/source/net/yacy/http/servlets/UrlProxyServlet.javaindex 826a8cd4d..bdddc4002 100644--- a/source/net/yacy/http/servlets/UrlProxyServlet.java+++ b/source/net/yacy/http/servlets/UrlProxyServlet.java@@ -156,6 +156,7 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {prop.put(HeaderFramework.CONNECTION_PROP_METHOD, request.getMethod()); // only needed for HTTPDeamon errormsg in case of blacklisted urlif (proxyurl.getQuery() != null) prop.put(HeaderFramework.CONNECTION_PROP_ARGS, proxyurl.getQuery());prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);+ prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);yacyRequestHeader.put(HeaderFramework.HOST, hostwithport );yacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath());diff --git a/source/net/yacy/http/servlets/YaCyProxyServlet.java b/source/net/yacy/http/servlets/YaCyProxyServlet.javaindex 2f5791783..7591a15b2 100644--- a/source/net/yacy/http/servlets/YaCyProxyServlet.java+++ b/source/net/yacy/http/servlets/YaCyProxyServlet.java@@ -126,6 +126,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);+ prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);yacyRequestHeader.put(HeaderFramework.HOST, hostwithport );yacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath());diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.javaindex b97b6be2f..b88439d3f 100644--- a/source/net/yacy/server/http/HTTPDProxyHandler.java+++ b/source/net/yacy/server/http/HTTPDProxyHandler.java@@ -66,6 +66,7 @@ import java.util.logging.FileHandler;import java.util.logging.Level;import java.util.logging.LogManager;import java.util.logging.Logger;+import javax.servlet.http.HttpServletRequest;import javax.servlet.http.HttpServletResponse;import net.yacy.cora.document.id.DigestURL;@@ -170,7 +171,7 @@ public final class HTTPDProxyHandler {yellowList = FileUtils.loadList(new File(f));log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");} else {- yellowList = new HashSet<String>();+ yellowList = null;}final String redirectorPath = sb.getConfig("externalRedirector", "");@@ -255,6 +256,8 @@ public final class HTTPDProxyHandler {}/**+ * Get a page from the remote server as proxy request for the client+ ** @param conProp a collection of properties about the connection, like URL* @param requestHeader The header lines of the connection from the request* @param respond the OutputStream to the client@@ -404,6 +407,16 @@ public final class HTTPDProxyHandler {}}+ /**+ * Get requested proxied page from the web+ *+ * @param conProp+ * @param url+ * @param requestHeader+ * @param cachedResponseHeader+ * @param respond+ * @param agent+ */private static void fulfillRequestFromWeb(final HashMap<String, Object> conProp, final DigestURL url, final RequestHeader requestHeader, final ResponseHeader cachedResponseHeader, final OutputStream respond, final ClientIdentification.Agent agent) {try {final int reqID = requestHeader.hashCode();@@ -412,7 +425,14 @@ public final class HTTPDProxyHandler {final String path = url.getPath(); // conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'final String args = url.getSearchpart(); // conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were givenfinal String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer- final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); // the ip from the connecting peer+ final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); // the http version for the request+ final String clienthttpVer; // the http version of the client connection+ final HttpServletRequest clientservletrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);+ if (clientservletrequest != null) {+ clienthttpVer = clientservletrequest.getProtocol();+ } else {+ clienthttpVer = null;+ }final int port = url.getPort();// resolve yacy and yacyh domains@@ -440,7 +460,6 @@ public final class HTTPDProxyHandler {try {client.GET(getUrl, false);if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());- conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader);int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());@@ -490,7 +509,7 @@ public final class HTTPDProxyHandler {HTTPDemon.sendRespondHeader(conProp,respond,- httpVer,+ clienthttpVer,statusCode,client.getHttpResponse().getStatusLine().toString(), // status textresponseHeader);@@ -634,7 +653,13 @@ public final class HTTPDProxyHandler {) throws IOException {final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);-+ final String clienthttpVer; // the http version of the client connection+ final HttpServletRequest clientservletrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);+ if (clientservletrequest != null) {+ clienthttpVer = clientservletrequest.getProtocol();+ } else {+ clienthttpVer = null;+ }// we respond on the request by using the cache, the cache is freshtry {prepareResponseHeader(cachedResponseHeader, httpVer);@@ -653,7 +678,7 @@ public final class HTTPDProxyHandler {cachedResponseHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(0));// send cached header with replaced date and added length- HTTPDemon.sendRespondHeader(conProp,respond,httpVer,304,cachedResponseHeader);+ HTTPDemon.sendRespondHeader(conProp,respond,clienthttpVer,304,cachedResponseHeader);//respondHeader(respond, "304 OK", cachedResponseHeader); // respond with 'not modified'} else {// unconditional request: send content of cache@@ -664,7 +689,7 @@ public final class HTTPDProxyHandler {cachedResponseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(cacheEntry.length));// send cached header with replaced date and added length- HTTPDemon.sendRespondHeader(conProp,respond,httpVer,203,cachedResponseHeader);+ HTTPDemon.sendRespondHeader(conProp,respond,clienthttpVer,203,cachedResponseHeader);//respondHeader(respond, "203 OK", cachedResponseHeader); // respond with 'non-authoritative'// send also the complete body now from the cache@@ -795,7 +820,7 @@ public final class HTTPDProxyHandler {if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) {forceConnectionClose(conProp);} else {- chunkedOut = new ChunkedOutputStream(respond);+ chunkedOut = new ChunkedOutputStream(respond);}responseHeader.remove(HeaderFramework.CONTENT_LENGTH);}diff --git a/source/net/yacy/server/http/HTTPDemon.java b/source/net/yacy/server/http/HTTPDemon.javaindex f2c3b0c75..5d2431eba 100644--- a/source/net/yacy/server/http/HTTPDemon.java+++ b/source/net/yacy/server/http/HTTPDemon.java@@ -128,7 +128,8 @@ public final class HTTPDemon {ByteArrayOutputStream o = null;try {// setting the proper http status message- String httpVersion = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); if (httpVersion == null) httpVersion = "HTTP/1.1";+ String httpVersion = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);+ if (httpVersion == null) httpVersion = HeaderFramework.HTTP_VERSION_1_1;if ((httpStatusText == null)||(httpStatusText.length()==0)) {//http1_1 includes http1_0 messagesif (HeaderFramework.http1_1.containsKey(Integer.toString(httpStatusCode)))