Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

change unused servletProperties entry CONNECTION_PROP_CLIENT_REQUEST_HEADER to use directly HttpServletRequest. This is used to get the http protocol version in HTTPDProxyHandler.fulfillRequestFromWeb() for error response to client. - adjust YaCyProxyServlet and UrlProxyServlet accordingly - use more http_version constants in headerframework and httpdeamon - equalize servlets (3) use of HeaderFramework.CONNECTION_PROP_HOST to HeaderFramework.HOST

reger3b47a07dd1d2
diff --git a/htroot/QuickCrawlLink_p.java b/htroot/QuickCrawlLink_p.java
index da3e81cd8..3e7158a44 100644
--- a/htroot/QuickCrawlLink_p.java
+++ b/htroot/QuickCrawlLink_p.java
@@ -70,7 +70,7 @@ public class QuickCrawlLink_p {
prop.put("mode", "0");
// get the http host header
- final String hostSocket = header.get(HeaderFramework.CONNECTION_PROP_HOST);
+ final String hostSocket = header.get(HeaderFramework.HOST);
//String host = hostSocket;
int port = 80;
diff --git a/htroot/YaCySearchPluginFF.java b/htroot/YaCySearchPluginFF.java
index 199bf1b73..9ddd0c5eb 100644
--- a/htroot/YaCySearchPluginFF.java
+++ b/htroot/YaCySearchPluginFF.java
@@ -47,7 +47,7 @@ public class YaCySearchPluginFF {
final serverObjects prop = new serverObjects();
// getting the http host header
- final String hostSocket = header.get(HeaderFramework.CONNECTION_PROP_HOST);
+ final String hostSocket = header.get(HeaderFramework.HOST);
String host = hostSocket;
int port = 80;
diff --git a/htroot/autoconfig.java b/htroot/autoconfig.java
index 4be45aab7..3ce7b313c 100644
--- a/htroot/autoconfig.java
+++ b/htroot/autoconfig.java
@@ -52,7 +52,7 @@ public class autoconfig {
final boolean yacyonly = env.getConfigBool(SwitchboardConstants.PROXY_YACY_ONLY, false);
// get the http host header
- final String hostSocket = header.get(HeaderFramework.CONNECTION_PROP_HOST);
+ final String hostSocket = header.get(HeaderFramework.HOST);
String host = hostSocket;
int port = 80;
diff --git a/source/net/yacy/cora/protocol/HeaderFramework.java b/source/net/yacy/cora/protocol/HeaderFramework.java
index 5048577c8..c013b9470 100644
--- a/source/net/yacy/cora/protocol/HeaderFramework.java
+++ b/source/net/yacy/cora/protocol/HeaderFramework.java
@@ -196,6 +196,9 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
}
/* PROPERTIES: General properties */
+ // Properties are used to internally store or remember header values and additional connection information
+ // One of the usages is in proxy operation to prepare header values to be set as header values upon connection
+ // * use of properties as header values is discouraged (e.g. as proxy transmits it as arbitrary headers) [2016-8-21]
public static final String CONNECTION_PROP_HTTP_VER = "HTTP";
public static final String CONNECTION_PROP_PROTOCOL = "PROTOCOL";
public static final String CONNECTION_PROP_HOST = "HOST";
@@ -207,16 +210,11 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
public static final String CONNECTION_PROP_ARGS = "ARGS";
public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP";
public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT";
- //public static final String CONNECTION_PROP_KEEP_ALIVE_COUNT = "KEEP-ALIVE_COUNT";
- //public static final String CONNECTION_PROP_REQUESTLINE = "REQUESTLINE";
- //public static final String CONNECTION_PROP_PREV_REQUESTLINE = "PREVREQUESTLINE";
public static final String CONNECTION_PROP_REQUEST_START = "REQUEST_START";
public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END";
- //public static final String CONNECTION_PROP_INPUTSTREAM = "INPUTSTREAM";
- //public static final String CONNECTION_PROP_OUTPUTSTREAM = "OUTPUTSTREAM";
/* PROPERTIES: Client -> Proxy */
- public static final String CONNECTION_PROP_CLIENT_REQUEST_HEADER = "CLIENT_REQUEST_HEADER";
+ public static final String CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST = "CLIENT_HTTPSERVLETREQUEST";
/* PROPERTIES: Proxy -> Client */
public static final String CONNECTION_PROP_PROXY_RESPOND_CODE = "PROXY_RESPOND_CODE";
@@ -546,7 +544,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
if (theHeader == null) throw new IllegalArgumentException();
// setting the http version if it was not already set
- if (httpVersion == null) httpVersion = "HTTP/1.0";
+ if (httpVersion == null) httpVersion = HTTP_VERSION_1_0;
// setting the status text if it was not already set
if ((httpStatusText == null)||(httpStatusText.length()==0)) {
diff --git a/source/net/yacy/http/servlets/UrlProxyServlet.java b/source/net/yacy/http/servlets/UrlProxyServlet.java
index 826a8cd4d..bdddc4002 100644
--- a/source/net/yacy/http/servlets/UrlProxyServlet.java
+++ b/source/net/yacy/http/servlets/UrlProxyServlet.java
@@ -156,6 +156,7 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
prop.put(HeaderFramework.CONNECTION_PROP_METHOD, request.getMethod()); // only needed for HTTPDeamon errormsg in case of blacklisted url
if (proxyurl.getQuery() != null) prop.put(HeaderFramework.CONNECTION_PROP_ARGS, proxyurl.getQuery());
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
+ prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);
yacyRequestHeader.put(HeaderFramework.HOST, hostwithport );
yacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath());
diff --git a/source/net/yacy/http/servlets/YaCyProxyServlet.java b/source/net/yacy/http/servlets/YaCyProxyServlet.java
index 2f5791783..7591a15b2 100644
--- a/source/net/yacy/http/servlets/YaCyProxyServlet.java
+++ b/source/net/yacy/http/servlets/YaCyProxyServlet.java
@@ -126,6 +126,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
+ prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);
yacyRequestHeader.put(HeaderFramework.HOST, hostwithport );
yacyRequestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath());
diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.java
index b97b6be2f..b88439d3f 100644
--- a/source/net/yacy/server/http/HTTPDProxyHandler.java
+++ b/source/net/yacy/server/http/HTTPDProxyHandler.java
@@ -66,6 +66,7 @@ import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.Logger;
+import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import net.yacy.cora.document.id.DigestURL;
@@ -170,7 +171,7 @@ public final class HTTPDProxyHandler {
yellowList = FileUtils.loadList(new File(f));
log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
} else {
- yellowList = new HashSet<String>();
+ yellowList = null;
}
final String redirectorPath = sb.getConfig("externalRedirector", "");
@@ -255,6 +256,8 @@ public final class HTTPDProxyHandler {
}
/**
+ * Get a page from the remote server as proxy request for the client
+ *
* @param conProp a collection of properties about the connection, like URL
* @param requestHeader The header lines of the connection from the request
* @param respond the OutputStream to the client
@@ -404,6 +407,16 @@ public final class HTTPDProxyHandler {
}
}
+ /**
+ * Get requested proxied page from the web
+ *
+ * @param conProp
+ * @param url
+ * @param requestHeader
+ * @param cachedResponseHeader
+ * @param respond
+ * @param agent
+ */
private static void fulfillRequestFromWeb(final HashMap<String, Object> conProp, final DigestURL url, final RequestHeader requestHeader, final ResponseHeader cachedResponseHeader, final OutputStream respond, final ClientIdentification.Agent agent) {
try {
final int reqID = requestHeader.hashCode();
@@ -412,7 +425,14 @@ public final class HTTPDProxyHandler {
final String path = url.getPath(); // conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
final String args = url.getSearchpart(); // conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
- final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); // the ip from the connecting peer
+ final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); // the http version for the request
+ final String clienthttpVer; // the http version of the client connection
+ final HttpServletRequest clientservletrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
+ if (clientservletrequest != null) {
+ clienthttpVer = clientservletrequest.getProtocol();
+ } else {
+ clienthttpVer = null;
+ }
final int port = url.getPort();
// resolve yacy and yacyh domains
@@ -440,7 +460,6 @@ public final class HTTPDProxyHandler {
try {
client.GET(getUrl, false);
if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
- conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader);
int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
@@ -490,7 +509,7 @@ public final class HTTPDProxyHandler {
HTTPDemon.sendRespondHeader(
conProp,
respond,
- httpVer,
+ clienthttpVer,
statusCode,
client.getHttpResponse().getStatusLine().toString(), // status text
responseHeader);
@@ -634,7 +653,13 @@ public final class HTTPDProxyHandler {
) throws IOException {
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
-
+ final String clienthttpVer; // the http version of the client connection
+ final HttpServletRequest clientservletrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
+ if (clientservletrequest != null) {
+ clienthttpVer = clientservletrequest.getProtocol();
+ } else {
+ clienthttpVer = null;
+ }
// we respond on the request by using the cache, the cache is fresh
try {
prepareResponseHeader(cachedResponseHeader, httpVer);
@@ -653,7 +678,7 @@ public final class HTTPDProxyHandler {
cachedResponseHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(0));
// send cached header with replaced date and added length
- HTTPDemon.sendRespondHeader(conProp,respond,httpVer,304,cachedResponseHeader);
+ HTTPDemon.sendRespondHeader(conProp,respond,clienthttpVer,304,cachedResponseHeader);
//respondHeader(respond, "304 OK", cachedResponseHeader); // respond with 'not modified'
} else {
// unconditional request: send content of cache
@@ -664,7 +689,7 @@ public final class HTTPDProxyHandler {
cachedResponseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(cacheEntry.length));
// send cached header with replaced date and added length
- HTTPDemon.sendRespondHeader(conProp,respond,httpVer,203,cachedResponseHeader);
+ HTTPDemon.sendRespondHeader(conProp,respond,clienthttpVer,203,cachedResponseHeader);
//respondHeader(respond, "203 OK", cachedResponseHeader); // respond with 'non-authoritative'
// send also the complete body now from the cache
@@ -795,7 +820,7 @@ public final class HTTPDProxyHandler {
if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) {
forceConnectionClose(conProp);
} else {
- chunkedOut = new ChunkedOutputStream(respond);
+ chunkedOut = new ChunkedOutputStream(respond);
}
responseHeader.remove(HeaderFramework.CONTENT_LENGTH);
}
diff --git a/source/net/yacy/server/http/HTTPDemon.java b/source/net/yacy/server/http/HTTPDemon.java
index f2c3b0c75..5d2431eba 100644
--- a/source/net/yacy/server/http/HTTPDemon.java
+++ b/source/net/yacy/server/http/HTTPDemon.java
@@ -128,7 +128,8 @@ public final class HTTPDemon {
ByteArrayOutputStream o = null;
try {
// setting the proper http status message
- String httpVersion = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); if (httpVersion == null) httpVersion = "HTTP/1.1";
+ String httpVersion = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
+ if (httpVersion == null) httpVersion = HeaderFramework.HTTP_VERSION_1_1;
if ((httpStatusText == null)||(httpStatusText.length()==0)) {
//http1_1 includes http1_0 messages
if (HeaderFramework.http1_1.containsKey(Integer.toString(httpStatusCode)))