Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

handle all references for client, response, request to be able to close them

sgaebelfc4275f901e5
diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java
index cda95c01c..834f4fdf1 100644
--- a/source/net/yacy/cora/protocol/http/HTTPClient.java
+++ b/source/net/yacy/cora/protocol/http/HTTPClient.java
@@ -170,14 +170,18 @@ public class HTTPClient {
private final static HttpClientBuilder clientBuilder = initClientBuilder();
private final RequestConfig.Builder reqConfBuilder;
private Set<Entry<String, String>> headers = null;
- private CloseableHttpResponse httpResponse = null;
- private HttpUriRequest currentRequest = null;
private long upbytes = 0L;
private String host = null;
private final long timeout;
private static ExecutorService executor = Executors
.newCachedThreadPool(new NamePrefixThreadFactory(HTTPClient.class.getSimpleName() + ".execute"));
+
+ /** these are the main variable to hold information and to take care of closing: */
+ private CloseableHttpClient client = null;
+ private CloseableHttpResponse httpResponse = null;
+ private HttpUriRequest currentRequest = null;
+
public HTTPClient(final ClientIdentification.Agent agent) {
super();
this.timeout = agent.clientTimeout;
@@ -214,6 +218,8 @@ public class HTTPClient {
final HttpClientBuilder builder = HttpClientBuilder.create();
builder.setConnectionManager(CONNECTION_MANAGER);
+ builder.setConnectionManagerShared(true);
+
builder.setDefaultRequestConfig(DFLTREQUESTCONFIG);
// UserAgent
@@ -429,15 +435,15 @@ public class HTTPClient {
public byte[] GETbytes(final MultiProtocolURL url, final String username, final String pass, final int maxBytes, final boolean concurrent) throws IOException {
final boolean localhost = Domains.isLocalhost(url.getHost());
final String urix = url.toNormalform(true);
- HttpGet httpGet = null;
+
try {
- httpGet = new HttpGet(urix);
+ this.currentRequest = new HttpGet(urix);
} catch (IllegalArgumentException e) {
throw new IOException(e.getMessage()); // can be caused at java.net.URI.create()
}
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (!localhost || pass == null) {
- return getContentBytes(httpGet, maxBytes, concurrent);
+ return getContentBytes(maxBytes, concurrent);
}
CredentialsProvider credsProvider = new BasicCredentialsProvider();
@@ -447,7 +453,7 @@ public class HTTPClient {
try (final CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
.setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) {
- this.httpResponse = httpclient.execute(httpGet);
+ this.httpResponse = httpclient.execute(this.currentRequest);
try {
HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) {
@@ -461,11 +467,8 @@ public class HTTPClient {
return getByteArray(httpEntity, maxBytes);
}
}
- } catch (final IOException e) {
- httpGet.abort();
- throw e;
} finally {
- this.httpResponse.close();
+ finish();
}
}
return null;
@@ -494,15 +497,15 @@ public class HTTPClient {
public void GET(final MultiProtocolURL url, final boolean concurrent) throws IOException {
if (this.currentRequest != null) throw new IOException("Client is in use!");
final String urix = url.toNormalform(true);
- HttpGet httpGet = null;
+
try {
- httpGet = new HttpGet(urix);
+ this.currentRequest = new HttpGet(urix);
} catch (IllegalArgumentException e) {
throw new IOException(e.getMessage()); // can be caused at java.net.URI.create()
}
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
- this.currentRequest = httpGet;
- execute(httpGet, concurrent);
+
+ execute(concurrent);
}
/**
@@ -524,11 +527,9 @@ public class HTTPClient {
* @throws IOException
*/
public HttpResponse HEADResponse(final MultiProtocolURL url, final boolean concurrent) throws IOException {
- final HttpHead httpHead = new HttpHead(url.toNormalform(true));
+ this.currentRequest = new HttpHead(url.toNormalform(true));
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
- execute(httpHead, concurrent);
- finish();
- ConnectionInfo.removeConnection(httpHead.hashCode());
+ execute(concurrent);
return this.httpResponse;
}
@@ -560,16 +561,15 @@ public class HTTPClient {
*/
public void POST(final MultiProtocolURL url, final InputStream instream, final long length, final boolean concurrent) throws IOException {
if (this.currentRequest != null) throw new IOException("Client is in use!");
- final HttpPost httpPost = new HttpPost(url.toNormalform(true));
+ this.currentRequest = new HttpPost(url.toNormalform(true));
String host = url.getHost();
if (host == null) host = Domains.LOCALHOST;
setHost(host); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
final NonClosingInputStreamEntity inputStreamEntity = new NonClosingInputStreamEntity(instream, length);
// statistics
this.upbytes = length;
- httpPost.setEntity(inputStreamEntity);
- this.currentRequest = httpPost;
- execute(httpPost, concurrent);
+ ((HttpPost) this.currentRequest).setEntity(inputStreamEntity);
+ execute(concurrent);
}
/**
@@ -615,7 +615,7 @@ public class HTTPClient {
*/
public byte[] POSTbytes(final MultiProtocolURL url, final String vhost, final Map<String, ContentBody> post,
final String userName, final String password, final boolean usegzip, final boolean concurrent) throws IOException {
- final HttpPost httpPost = new HttpPost(url.toNormalform(true));
+ this.currentRequest = new HttpPost(url.toNormalform(true));
final boolean localhost = Domains.isLocalhost(url.getHost());
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (vhost == null) setHost(Domains.LOCALHOST);
@@ -627,13 +627,13 @@ public class HTTPClient {
this.upbytes = multipartEntity.getContentLength();
if (usegzip) {
- httpPost.setEntity(new GzipCompressingEntity(multipartEntity));
+ ((HttpPost) this.currentRequest).setEntity(new GzipCompressingEntity(multipartEntity));
} else {
- httpPost.setEntity(multipartEntity);
+ ((HttpPost) this.currentRequest).setEntity(multipartEntity);
}
if (!localhost || password == null) {
- return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent);
+ return getContentBytes(Integer.MAX_VALUE, concurrent);
}
final CredentialsProvider credsProvider = new BasicCredentialsProvider();
@@ -643,7 +643,7 @@ public class HTTPClient {
try (final CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
.setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) {
- this.httpResponse = httpclient.execute(httpPost);
+ this.httpResponse = httpclient.execute(this.currentRequest);
try {
HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) {
@@ -652,7 +652,7 @@ public class HTTPClient {
}
}
} finally {
- this.httpResponse.close();
+ finish();
}
}
return null;
@@ -820,6 +820,9 @@ public class HTTPClient {
EntityUtils.consumeQuietly(this.httpResponse.getEntity());
this.httpResponse.close();
}
+ if (this.client != null) {
+ client.close();
+ }
} finally {
if (this.currentRequest != null) {
ConnectionInfo.removeConnection(this.currentRequest.hashCode());
@@ -829,9 +832,9 @@ public class HTTPClient {
}
}
- private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final int maxBytes, final boolean concurrent) throws IOException {
+ private byte[] getContentBytes(final int maxBytes, final boolean concurrent) throws IOException {
try {
- execute(httpUriRequest, concurrent);
+ execute(concurrent);
if (this.httpResponse == null) return null;
// get the response body
final HttpEntity httpEntity = this.httpResponse.getEntity();
@@ -846,28 +849,24 @@ public class HTTPClient {
return getByteArray(httpEntity, maxBytes);
}
}
- } catch (final IOException e) {
- httpUriRequest.abort();
- throw e;
} finally {
- if (this.httpResponse != null) this.httpResponse.close();
- ConnectionInfo.removeConnection(httpUriRequest.hashCode());
+ finish();
}
return null;
}
- private void execute(final HttpUriRequest httpUriRequest, final boolean concurrent) throws IOException {
+ private void execute(final boolean concurrent) throws IOException {
final HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(reqConfBuilder.build());
if (this.host != null)
context.setTargetHost(new HttpHost(this.host));
- setHeaders(httpUriRequest);
+ setHeaders();
// statistics
- storeConnectionInfo(httpUriRequest);
+ storeConnectionInfo();
// execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF
- if (httpUriRequest instanceof HttpEntityEnclosingRequest) {
- final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) httpUriRequest;
+ if (this.currentRequest instanceof HttpEntityEnclosingRequest) {
+ final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) this.currentRequest;
final HttpEntity entity = hrequest.getEntity();
assert entity != null;
//assert !entity.isChunked();
@@ -876,16 +875,17 @@ public class HTTPClient {
}
final String initialThreadName = Thread.currentThread().getName();
- Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI());
+ final String uri = this.currentRequest.getURI().toString();
+ Thread.currentThread().setName("HTTPClient-" + uri);
final long time = System.currentTimeMillis();
try {
+ this.client = clientBuilder.build();
if (concurrent) {
FutureTask<CloseableHttpResponse> t = new FutureTask<CloseableHttpResponse>(new Callable<CloseableHttpResponse>() {
@Override
public CloseableHttpResponse call() throws ClientProtocolException, IOException {
- final CloseableHttpClient client = clientBuilder.build();
- CloseableHttpResponse response = client.execute(httpUriRequest, context);
+ CloseableHttpResponse response = client.execute(currentRequest, context);
return response;
}
});
@@ -896,20 +896,18 @@ public class HTTPClient {
throw e.getCause();
} catch (Throwable e) {}
try {t.cancel(true);} catch (Throwable e) {}
- if (this.httpResponse == null) throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + httpUriRequest.getURI().toString());
+ if (this.httpResponse == null) {
+ throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + uri);
+ }
} else {
- final CloseableHttpClient client = clientBuilder.build();
- this.httpResponse = client.execute(httpUriRequest, context);
+ this.httpResponse = client.execute(this.currentRequest, context);
}
this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time));
} catch (final Throwable e) {
- ConnectionInfo.removeConnection(httpUriRequest.hashCode());
- httpUriRequest.abort();
- if (this.httpResponse != null) this.httpResponse.close();
- //e.printStackTrace();
+ finish();
throw new IOException("Client can't execute: "
+ (e.getCause() == null ? e.getMessage() : e.getCause().getMessage())
- + " duration=" + Long.toString(System.currentTimeMillis() - time) + " for url " + httpUriRequest.getURI().toString());
+ + " duration=" + Long.toString(System.currentTimeMillis() - time) + " for url " + uri);
} finally {
/* Restore the thread initial name */
Thread.currentThread().setName(initialThreadName);
@@ -973,25 +971,25 @@ public class HTTPClient {
}
}
- private void setHeaders(final HttpUriRequest httpUriRequest) {
+ private void setHeaders() {
if (this.headers != null) {
for (final Entry<String, String> entry : this.headers) {
- httpUriRequest.setHeader(entry.getKey(),entry.getValue());
+ this.currentRequest.setHeader(entry.getKey(),entry.getValue());
}
}
- if (this.host != null) httpUriRequest.setHeader(HTTP.TARGET_HOST, this.host);
- httpUriRequest.setHeader(HTTP.CONN_DIRECTIVE, "close"); // don't keep alive, prevent CLOSE_WAIT state
+ if (this.host != null) this.currentRequest.setHeader(HTTP.TARGET_HOST, this.host);
+ this.currentRequest.setHeader(HTTP.CONN_DIRECTIVE, "close"); // don't keep alive, prevent CLOSE_WAIT state
}
- private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
- final int port = httpUriRequest.getURI().getPort();
- final String thost = httpUriRequest.getURI().getHost();
+ private void storeConnectionInfo() {
+ final int port = this.currentRequest.getURI().getPort();
+ final String thost = this.currentRequest.getURI().getHost();
//assert thost != null : "uri = " + httpUriRequest.getURI().toString();
ConnectionInfo.addConnection(new ConnectionInfo(
- httpUriRequest.getURI().getScheme(),
+ this.currentRequest.getURI().getScheme(),
port == -1 ? thost : thost + ":" + port,
- httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(),
- httpUriRequest.hashCode(),
+ this.currentRequest.getMethod() + " " + this.currentRequest.getURI().getPath(),
+ this.currentRequest.hashCode(),
System.currentTimeMillis(),
this.upbytes));
}