Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
always use HTTPClient by 'try with resources' pattern to free up resources
sgaebelcdf901270cb2
diff --git a/htroot/IndexImportMediawiki_p.java b/htroot/IndexImportMediawiki_p.javaindex 0b95382d7..b45a76a5e 100644--- a/htroot/IndexImportMediawiki_p.java+++ b/htroot/IndexImportMediawiki_p.java@@ -196,29 +196,29 @@ public class IndexImportMediawiki_p {* @return the last modified date for the file at fileURL, or 0L when unknown or when an error occurred*/private static long getLastModified(MultiProtocolURL fileURL) {- long lastModified = 0l;try {if (fileURL.isHTTP() || fileURL.isHTTPS()) {/* http(s) : we do not use MultiprotocolURL.lastModified() which always returns 0L for these protocols */- HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);- HttpResponse headResponse = httpClient.HEADResponse(fileURL, false);- if (headResponse != null && headResponse.getStatusLine() != null- && headResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {- Header lastModifiedHeader = headResponse- .getFirstHeader(HeaderFramework.LAST_MODIFIED);- if (lastModifiedHeader != null) {- Date lastModifiedDate = HeaderFramework.parseHTTPDate(lastModifiedHeader.getValue());- if(lastModifiedDate != null) {- lastModified = lastModifiedDate.getTime();- }- }- }+ try (HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {+ HttpResponse headResponse = httpClient.HEADResponse(fileURL, false);+ if (headResponse != null && headResponse.getStatusLine() != null+ && headResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {+ Header lastModifiedHeader = headResponse+ .getFirstHeader(HeaderFramework.LAST_MODIFIED);+ if (lastModifiedHeader != null) {+ Date lastModifiedDate = HeaderFramework.parseHTTPDate(lastModifiedHeader.getValue());+ if(lastModifiedDate != null) {+ return lastModifiedDate.getTime();+ }+ }+ }+ }} else {- lastModified = fileURL.lastModified();+ return fileURL.lastModified();}} catch (IOException ignored) {ConcurrentLog.warn("IndexImportMediawiki_p", "Could not retrieve last modified date for dump file at " + fileURL);}- return lastModified;+ return 0l;}}diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.javaindex 7f4bf9a2c..92ab41a52 100644--- a/source/net/yacy/cora/document/id/MultiProtocolURL.java+++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java@@ -2538,7 +2538,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolUreturn new ByteArrayInputStream(b);}if (isHTTP() || isHTTPS()) {- final HTTPClient client = new HTTPClient(agent);+ try (final HTTPClient client = new HTTPClient(agent)){client.setHost(getHost());client.GET(this, false);if (client.getStatusCode() != HttpStatus.SC_OK) {@@ -2546,6 +2546,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU"\nServer returned status: " + client.getHttpResponse().getStatusLine());}return new HTTPInputStream(client);+ }}return null;@@ -2562,9 +2563,10 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolUreturn b;}if (isHTTP() || isHTTPS()) {- final HTTPClient client = new HTTPClient(agent);- client.setHost(getHost());- return client.GETbytes(this, username, pass, false);+ try (final HTTPClient client = new HTTPClient(agent)) {+ client.setHost(getHost());+ return client.GETbytes(this, username, pass, false);+ }}return null;diff --git a/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java b/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.javaindex 0bb06fbba..d61a9deed 100644--- a/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java+++ b/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java@@ -297,10 +297,9 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector implemeString searchurl = this.parseSearchTemplate(baseurl, searchTerms, startIndex, count);try {DigestURL aurl = new DigestURL(searchurl);- try {+ try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {this.lastaccesstime = System.currentTimeMillis();- final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);byte[] result = httpClient.GETbytes(aurl, null, null, false);if(result == null) {diff --git a/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java b/source/net/yacy/cora/federate/opensearch/SRURSSConnector.javaindex ffaadf458..2f2892267 100644--- a/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java+++ b/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java@@ -121,8 +121,9 @@ public class SRURSSConnector {parts.put("resource", UTF8.StringBody(global ? "global" : "local"));parts.put("nav", UTF8.StringBody("none"));// result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);- final HTTPClient httpClient = new HTTPClient(agent);- result = httpClient.POSTbytes(new MultiProtocolURL(rssSearchServiceURL), uri.getHost(), parts, false, false);+ try (final HTTPClient httpClient = new HTTPClient(agent)) {+ result = httpClient.POSTbytes(new MultiProtocolURL(rssSearchServiceURL), uri.getHost(), parts, false, false);+ }final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);if (reader == null) {diff --git a/source/net/yacy/cora/federate/yacy/api/Network.java b/source/net/yacy/cora/federate/yacy/api/Network.javaindex 876d1ffa7..486b1b917 100644--- a/source/net/yacy/cora/federate/yacy/api/Network.java+++ b/source/net/yacy/cora/federate/yacy/api/Network.java@@ -49,7 +49,7 @@ public class Network {*/public static Peers getNetwork(final String address) throws IOException {Peers peers = new Peers();- final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);+ try (final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {final byte[] content = httpclient.GETbytes("http://" + address + "/Network.xml?page=1&maxCount=1000&ip=", null, null, false);ByteArrayInputStream bais = new ByteArrayInputStream(content);Document doc = null;@@ -74,6 +74,7 @@ public class Network {//log.info(peer.toString());}}+ }return peers;}diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.javaindex 9c4497293..acdc452e6 100644--- a/source/net/yacy/crawler/retrieval/HTTPLoader.java+++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java@@ -140,123 +140,124 @@ public final class HTTPLoader {final RequestHeader requestHeader = createRequestheader(request, agent);// HTTP-Client- final HTTPClient client = new HTTPClient(agent);- client.setRedirecting(false); // we want to handle redirection- // ourselves, so we don't index pages- // twice- client.setTimout(this.socketTimeout);- client.setHeader(requestHeader.entrySet());-- // send request- client.GET(url, false);- final StatusLine statusline = client.getHttpResponse().getStatusLine();- final int statusCode = statusline.getStatusCode();- final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());- String requestURLString = request.url().toNormalform(true);-- // check redirection- if (statusCode > 299 && statusCode < 310) {- client.close();-- final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, statusline, responseHeader, requestURLString);-- if (this.sb.getConfigBool(SwitchboardConstants.CRAWLER_FOLLOW_REDIRECTS, true)) {- // we have two use cases here: loading from a crawl or just- // loading the url. Check this:- if (profile != null && !CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) {- // put redirect url on the crawler queue to repeat a- // double-check- /* We have to clone the request instance and not to modify directly its URL,- * otherwise the stackCrawl() function would reject it, because detecting it as already in the activeWorkerEntries */- Request redirectedRequest = new Request(request.initiator(),- redirectionUrl,- request.referrerhash(),- request.name(),- request.appdate(),- request.profileHandle(),- request.depth(),- request.timezoneOffset());- String rejectReason = this.sb.crawlStacker.stackCrawl(redirectedRequest);- if(rejectReason != null) {- throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted. Reason : " + rejectReason);+ try (final HTTPClient client = new HTTPClient(agent)) {+ client.setRedirecting(false); // we want to handle redirection+ // ourselves, so we don't index pages+ // twice+ client.setTimout(this.socketTimeout);+ client.setHeader(requestHeader.entrySet());++ // send request+ client.GET(url, false);+ final StatusLine statusline = client.getHttpResponse().getStatusLine();+ final int statusCode = statusline.getStatusCode();+ final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());+ String requestURLString = request.url().toNormalform(true);++ // check redirection+ if (statusCode > 299 && statusCode < 310) {+ client.close();++ final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, statusline, responseHeader, requestURLString);++ if (this.sb.getConfigBool(SwitchboardConstants.CRAWLER_FOLLOW_REDIRECTS, true)) {+ // we have two use cases here: loading from a crawl or just+ // loading the url. Check this:+ if (profile != null && !CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) {+ // put redirect url on the crawler queue to repeat a+ // double-check+ /* We have to clone the request instance and not to modify directly its URL,+ * otherwise the stackCrawl() function would reject it, because detecting it as already in the activeWorkerEntries */+ Request redirectedRequest = new Request(request.initiator(),+ redirectionUrl,+ request.referrerhash(),+ request.name(),+ request.appdate(),+ request.profileHandle(),+ request.depth(),+ request.timezoneOffset());+ String rejectReason = this.sb.crawlStacker.stackCrawl(redirectedRequest);+ if(rejectReason != null) {+ throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted. Reason : " + rejectReason);+ }+ // in the end we must throw an exception (even if this is+ // not an error, just to abort the current process+ throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " to "+ + redirectionUrl.toNormalform(false) + " placed on crawler queue for double-check");}- // in the end we must throw an exception (even if this is- // not an error, just to abort the current process- throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " to "- + redirectionUrl.toNormalform(false) + " placed on crawler queue for double-check");- }-- // if we are already doing a shutdown we don't need to retry- // crawling- if (Thread.currentThread().isInterrupted()) {- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile,- FailCategory.FINAL_LOAD_CONTEXT, "server shutdown", statusCode);- throw new IOException(- "CRAWLER Redirect of URL=" + requestURLString + " aborted because of server shutdown.$");- }-- // check if the redirected URL is the same as the requested URL- // this shortcuts a time-out using retryCount- if (redirectionUrl.equals(url)) {- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "redirect to same url", -1);- throw new IOException( "retry counter exceeded for URL " + request.url().toString() + ". Processing aborted.$");- }-- // retry crawling with new url- request.redirectURL(redirectionUrl);- return openInputStream(request, profile, retryCount - 1, maxFileSize, blacklistType, agent);- }- // we don't want to follow redirects- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "redirection not wanted", statusCode);- throw new IOException("REJECTED UNWANTED REDIRECTION '" + statusline + "' for URL '" + requestURLString + "'$");- } else if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NON_AUTHORITATIVE_INFORMATION) {- // the transfer is ok-- /*- * When content is not large (less than Response.CRAWLER_MAX_SIZE_TO_CACHE), we have better cache it if cache is enabled and url is not local- */- long contentLength = client.getHttpResponse().getEntity().getContentLength();- InputStream contentStream;- if (profile != null && profile.storeHTCache() && contentLength > 0 && contentLength < (Response.CRAWLER_MAX_SIZE_TO_CACHE) && !url.isLocal()) {- byte[] content = null;- try {- content = HTTPClient.getByteArray(client.getHttpResponse().getEntity(), maxFileSize);- Cache.store(url, responseHeader, content);- } catch (final IOException e) {- this.log.warn("cannot write " + url + " to Cache (3): " + e.getMessage(), e);- } finally {- client.close();- }-- contentStream = new ByteArrayInputStream(content);- } else {- /*- * Content length may already be known now : check it before opening a stream- */- if (maxFileSize >= 0 && contentLength > maxFileSize) {- throw new IOException("Content to download exceed maximum value of " + maxFileSize + " bytes");++ // if we are already doing a shutdown we don't need to retry+ // crawling+ if (Thread.currentThread().isInterrupted()) {+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile,+ FailCategory.FINAL_LOAD_CONTEXT, "server shutdown", statusCode);+ throw new IOException(+ "CRAWLER Redirect of URL=" + requestURLString + " aborted because of server shutdown.$");+ }++ // check if the redirected URL is the same as the requested URL+ // this shortcuts a time-out using retryCount+ if (redirectionUrl.equals(url)) {+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "redirect to same url", -1);+ throw new IOException( "retry counter exceeded for URL " + request.url().toString() + ". Processing aborted.$");+ }++ // retry crawling with new url+ request.redirectURL(redirectionUrl);+ return openInputStream(request, profile, retryCount - 1, maxFileSize, blacklistType, agent);}+ // we don't want to follow redirects+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "redirection not wanted", statusCode);+ throw new IOException("REJECTED UNWANTED REDIRECTION '" + statusline + "' for URL '" + requestURLString + "'$");+ } else if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NON_AUTHORITATIVE_INFORMATION) {+ // the transfer is ok+/*- * Create a HTTPInputStream delegating to- * client.getContentstream(). Close method will ensure client is- * properly closed.+ * When content is not large (less than Response.CRAWLER_MAX_SIZE_TO_CACHE), we have better cache it if cache is enabled and url is not local*/- contentStream = new HTTPInputStream(client);- /* Anticipated content length may not be already known or incorrect : let's apply now the same eventual content size restriction as when loading in a byte array */- if(maxFileSize >= 0) {- contentStream = new StrictLimitInputStream(contentStream, maxFileSize,- "Content to download exceed maximum value of " + Formatter.bytesToString(maxFileSize));+ long contentLength = client.getHttpResponse().getEntity().getContentLength();+ InputStream contentStream;+ if (profile != null && profile.storeHTCache() && contentLength > 0 && contentLength < (Response.CRAWLER_MAX_SIZE_TO_CACHE) && !url.isLocal()) {+ byte[] content = null;+ try {+ content = HTTPClient.getByteArray(client.getHttpResponse().getEntity(), maxFileSize);+ Cache.store(url, responseHeader, content);+ } catch (final IOException e) {+ this.log.warn("cannot write " + url + " to Cache (3): " + e.getMessage(), e);+ } finally {+ client.close();+ }++ contentStream = new ByteArrayInputStream(content);+ } else {+ /*+ * Content length may already be known now : check it before opening a stream+ */+ if (maxFileSize >= 0 && contentLength > maxFileSize) {+ throw new IOException("Content to download exceed maximum value of " + maxFileSize + " bytes");+ }+ /*+ * Create a HTTPInputStream delegating to+ * client.getContentstream(). Close method will ensure client is+ * properly closed.+ */+ contentStream = new HTTPInputStream(client);+ /* Anticipated content length may not be already known or incorrect : let's apply now the same eventual content size restriction as when loading in a byte array */+ if(maxFileSize >= 0) {+ contentStream = new StrictLimitInputStream(contentStream, maxFileSize,+ "Content to download exceed maximum value of " + Formatter.bytesToString(maxFileSize));+ }}++ return new StreamResponse(new Response(request, requestHeader, responseHeader, profile, false, null), contentStream);+ } else {+ client.close();+ // if the response has not the right response type then reject file+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile,+ FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode);+ throw new IOException("REJECTED WRONG STATUS TYPE '" + statusline+ + "' for URL '" + requestURLString + "'$");}-- return new StreamResponse(new Response(request, requestHeader, responseHeader, profile, false, null), contentStream);- } else {- client.close();- // if the response has not the right response type then reject file- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile,- FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode);- throw new IOException("REJECTED WRONG STATUS TYPE '" + statusline- + "' for URL '" + requestURLString + "'$");}}@@ -364,90 +365,91 @@ public final class HTTPLoader {final RequestHeader requestHeader = createRequestheader(request, agent);// HTTP-Client- final HTTPClient client = new HTTPClient(agent);- client.setRedirecting(false); // we want to handle redirection ourselves, so we don't index pages twice- client.setTimout(this.socketTimeout);- client.setHeader(requestHeader.entrySet());-- // send request- final byte[] responseBody = client.GETbytes(url, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), maxFileSize, false);- final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();- final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());- String requestURLString = request.url().toNormalform(true);-- // check redirection- if (statusCode > 299 && statusCode < 310) {-- final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, client.getHttpResponse().getStatusLine(),- responseHeader, requestURLString);-- if (this.sb.getConfigBool(SwitchboardConstants.CRAWLER_FOLLOW_REDIRECTS, true)) {- // we have two use cases here: loading from a crawl or just loading the url. Check this:- if (profile != null && !CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) {- // put redirect url on the crawler queue to repeat a double-check- /* We have to clone the request instance and not to modify directly its URL,- * otherwise the stackCrawl() function would reject it, because detecting it as already in the activeWorkerEntries */- Request redirectedRequest = new Request(request.initiator(),- redirectionUrl,- request.referrerhash(),- request.name(),- request.appdate(),- request.profileHandle(),- request.depth(),- request.timezoneOffset());- String rejectReason = this.sb.crawlStacker.stackCrawl(redirectedRequest);- // in the end we must throw an exception (even if this is not an error, just to abort the current process- if(rejectReason != null) {- throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted. Reason : " + rejectReason);+ try (final HTTPClient client = new HTTPClient(agent)) {+ client.setRedirecting(false); // we want to handle redirection ourselves, so we don't index pages twice+ client.setTimout(this.socketTimeout);+ client.setHeader(requestHeader.entrySet());++ // send request+ final byte[] responseBody = client.GETbytes(url, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), maxFileSize, false);+ final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();+ final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());+ String requestURLString = request.url().toNormalform(true);++ // check redirection+ if (statusCode > 299 && statusCode < 310) {++ final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, client.getHttpResponse().getStatusLine(),+ responseHeader, requestURLString);++ if (this.sb.getConfigBool(SwitchboardConstants.CRAWLER_FOLLOW_REDIRECTS, true)) {+ // we have two use cases here: loading from a crawl or just loading the url. Check this:+ if (profile != null && !CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) {+ // put redirect url on the crawler queue to repeat a double-check+ /* We have to clone the request instance and not to modify directly its URL,+ * otherwise the stackCrawl() function would reject it, because detecting it as already in the activeWorkerEntries */+ Request redirectedRequest = new Request(request.initiator(),+ redirectionUrl,+ request.referrerhash(),+ request.name(),+ request.appdate(),+ request.profileHandle(),+ request.depth(),+ request.timezoneOffset());+ String rejectReason = this.sb.crawlStacker.stackCrawl(redirectedRequest);+ // in the end we must throw an exception (even if this is not an error, just to abort the current process+ if(rejectReason != null) {+ throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted. Reason : " + rejectReason);+ }+ throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " to " + redirectionUrl.toNormalform(false) + " placed on crawler queue for double-check");}- throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " to " + redirectionUrl.toNormalform(false) + " placed on crawler queue for double-check");++ // if we are already doing a shutdown we don't need to retry crawling+ if (Thread.currentThread().isInterrupted()) {+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_LOAD_CONTEXT, "server shutdown", statusCode);+ throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted because of server shutdown.$");+ }++ // retry crawling with new url+ request.redirectURL(redirectionUrl);+ return load(request, profile, retryCount - 1, maxFileSize, blacklistType, agent);}-- // if we are already doing a shutdown we don't need to retry crawling- if (Thread.currentThread().isInterrupted()) {- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_LOAD_CONTEXT, "server shutdown", statusCode);- throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted because of server shutdown.$");+ // we don't want to follow redirects+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "redirection not wanted", statusCode);+ throw new IOException("REJECTED UNWANTED REDIRECTION '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");+ } else if (responseBody == null) {+ // no response, reject file+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "no response body", statusCode);+ throw new IOException("REJECTED EMPTY RESPONSE BODY '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");+ } else if (statusCode == 200 || statusCode == 203) {+ // the transfer is ok++ // we write the new cache entry to file system directly+ final long contentLength = responseBody.length;+ ByteCount.addAccountCount(ByteCount.CRAWLER, contentLength);++ // check length again in case it was not possible to get the length before loading+ if (maxFileSize >= 0 && contentLength > maxFileSize) {+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "file size limit exceeded", statusCode);+ throw new IOException("REJECTED URL " + request.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + maxFileSize + " bytes. (GET)$");}-- // retry crawling with new url- request.redirectURL(redirectionUrl);- return load(request, profile, retryCount - 1, maxFileSize, blacklistType, agent);- }- // we don't want to follow redirects- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "redirection not wanted", statusCode);- throw new IOException("REJECTED UNWANTED REDIRECTION '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");- } else if (responseBody == null) {- // no response, reject file- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "no response body", statusCode);- throw new IOException("REJECTED EMPTY RESPONSE BODY '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");- } else if (statusCode == 200 || statusCode == 203) {- // the transfer is ok-- // we write the new cache entry to file system directly- final long contentLength = responseBody.length;- ByteCount.addAccountCount(ByteCount.CRAWLER, contentLength);-- // check length again in case it was not possible to get the length before loading- if (maxFileSize >= 0 && contentLength > maxFileSize) {- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "file size limit exceeded", statusCode);- throw new IOException("REJECTED URL " + request.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + maxFileSize + " bytes. (GET)$");++ // create a new cache entry+ response = new Response(+ request,+ requestHeader,+ responseHeader,+ profile,+ false,+ responseBody+ );++ return response;+ } else {+ // if the response has not the right response type then reject file+ this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode);+ throw new IOException("REJECTED WRONG STATUS TYPE '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");}-- // create a new cache entry- response = new Response(- request,- requestHeader,- responseHeader,- profile,- false,- responseBody- );-- return response;- } else {- // if the response has not the right response type then reject file- this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode);- throw new IOException("REJECTED WRONG STATUS TYPE '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");}}@@ -484,9 +486,9 @@ public final class HTTPLoader {requestHeader.put(HeaderFramework.ACCEPT_CHARSET, DEFAULT_CHARSET);requestHeader.put(HeaderFramework.ACCEPT_ENCODING, DEFAULT_ENCODING);- final HTTPClient client = new HTTPClient(agent);- client.setTimout(20000);- client.setHeader(requestHeader.entrySet());+ try (final HTTPClient client = new HTTPClient(agent)) {+ client.setTimout(20000);+ client.setHeader(requestHeader.entrySet());final byte[] responseBody = client.GETbytes(request.url(), null, null, false);final int code = client.getHttpResponse().getStatusLine().getStatusCode();final ResponseHeader header = new ResponseHeader(code, client.getHttpResponse().getAllHeaders());@@ -539,6 +541,7 @@ public final class HTTPLoader {// if the response has not the right response type then reject filethrow new IOException("REJECTED WRONG STATUS TYPE '" + client.getHttpResponse().getStatusLine() + "' for URL " + request.url().toString());}+ }return response;}diff --git a/source/net/yacy/data/WorkTables.java b/source/net/yacy/data/WorkTables.javaindex 679c33057..0af5baf55 100644--- a/source/net/yacy/data/WorkTables.java+++ b/source/net/yacy/data/WorkTables.java@@ -327,50 +327,53 @@ public class WorkTables extends Tables {* @return a map of the called urls and the http status code of the api call or -1 if any other IOException occurred*/public Map<String, Integer> execAPICalls(String host, int port, Collection<String> pks, final String username, final String pass) {- // now call the api URLs and store the result status- final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);- client.setTimout(120000);- Tables.Row row;LinkedHashMap<String, Integer> l = new LinkedHashMap<String, Integer>();- for (final String pk: pks) {- row = null;- try {- row = select(WorkTables.TABLE_API_NAME, UTF8.getBytes(pk));- } catch (final IOException e) {- ConcurrentLog.logException(e);- } catch (final SpaceExceededException e) {- ConcurrentLog.logException(e);- }- if (row == null) continue;- String theapicall = UTF8.String(row.get(WorkTables.TABLE_API_COL_URL)) + "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK());- try {- MultiProtocolURL url = new MultiProtocolURL("http", host, port, theapicall);- final Map<String, String> attributes = url.getAttributes();- final boolean isTokenProtectedAPI = attributes.containsKey(TransactionManager.TRANSACTION_TOKEN_PARAM);- // use 4 param MultiProtocolURL to allow api_row_url with searchpart (like url?p=a&p2=b ) in client.GETbytes()- if (theapicall.length() > 1000 || isTokenProtectedAPI) {- // use a POST to execute the call- execPostAPICall(host, port, username, pass, client, l, url, isTokenProtectedAPI);- } else {- // use a GET to execute the call- ConcurrentLog.info("WorkTables", "executing url: " + url.toNormalform(true));- try {- client.GETbytes(url, username, pass, false); // use GETbytes(MultiProtocolURL,..) form to allow url in parameter (&url=path%- if(client.getStatusCode() == HttpStatus.SC_METHOD_NOT_ALLOWED) {- /* GET method not allowed (HTTP 450 status) : this may be an old API entry,- * now restricted to HTTP POST and requiring a transaction token. We try now with POST. */- execPostAPICall(host, port, username, pass, client, l, url, true);- } else {- l.put(url.toNormalform(true), client.getStatusCode());+ // now call the api URLs and store the result status+ try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {+ client.setTimout(120000);+ Tables.Row row;+ for (final String pk: pks) {+ row = null;+ try {+ row = select(WorkTables.TABLE_API_NAME, UTF8.getBytes(pk));+ } catch (final IOException e) {+ ConcurrentLog.logException(e);+ } catch (final SpaceExceededException e) {+ ConcurrentLog.logException(e);+ }+ if (row == null) continue;+ String theapicall = UTF8.String(row.get(WorkTables.TABLE_API_COL_URL)) + "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK());+ try {+ MultiProtocolURL url = new MultiProtocolURL("http", host, port, theapicall);+ final Map<String, String> attributes = url.getAttributes();+ final boolean isTokenProtectedAPI = attributes.containsKey(TransactionManager.TRANSACTION_TOKEN_PARAM);+ // use 4 param MultiProtocolURL to allow api_row_url with searchpart (like url?p=a&p2=b ) in client.GETbytes()+ if (theapicall.length() > 1000 || isTokenProtectedAPI) {+ // use a POST to execute the call+ execPostAPICall(host, port, username, pass, client, l, url, isTokenProtectedAPI);+ } else {+ // use a GET to execute the call+ ConcurrentLog.info("WorkTables", "executing url: " + url.toNormalform(true));+ try {+ client.GETbytes(url, username, pass, false); // use GETbytes(MultiProtocolURL,..) form to allow url in parameter (&url=path%+ if(client.getStatusCode() == HttpStatus.SC_METHOD_NOT_ALLOWED) {+ /* GET method not allowed (HTTP 450 status) : this may be an old API entry,+ * now restricted to HTTP POST and requiring a transaction token. We try now with POST. */+ execPostAPICall(host, port, username, pass, client, l, url, true);+ } else {+ l.put(url.toNormalform(true), client.getStatusCode());+ }+ } catch (final IOException e) {+ ConcurrentLog.logException(e);+ l.put(url.toString(), -1);}- } catch (final IOException e) {- ConcurrentLog.logException(e);- l.put(url.toString(), -1);}+ } catch (MalformedURLException ex) {+ ConcurrentLog.warn("APICALL", "wrong url in apicall " + theapicall);}- } catch (MalformedURLException ex) {- ConcurrentLog.warn("APICALL", "wrong url in apicall " + theapicall);}+ } catch (IOException e) {+ ConcurrentLog.logException(e);}return l;}@@ -447,11 +450,10 @@ public class WorkTables extends Tables {*/public static int execGetAPICall(String host, int port, String path, byte[] pk, final String username, final String pass) {// now call the api URLs and store the result status- final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);- client.setTimout(120000);String url = "http://" + host + ":" + port + path;if (pk != null) url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(pk);- try {+ try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {+ client.setTimout(120000);client.GETbytes(url, username, pass, false);return client.getStatusCode();} catch (final IOException e) {diff --git a/source/net/yacy/document/parser/sitemapParser.java b/source/net/yacy/document/parser/sitemapParser.javaindex 5486635c4..be52f72e7 100644--- a/source/net/yacy/document/parser/sitemapParser.java+++ b/source/net/yacy/document/parser/sitemapParser.java@@ -114,9 +114,8 @@ public class sitemapParser extends AbstractParser implements Parser {public static SitemapReader parse(final DigestURL sitemapURL, final ClientIdentification.Agent agent) throws IOException {// download documentConcurrentLog.info("SitemapReader", "loading sitemap from " + sitemapURL.toNormalform(true));- final HTTPClient client = new HTTPClient(agent);// client.setHeader(requestHeader.entrySet());- try {+ try (final HTTPClient client = new HTTPClient(agent)) {client.GET(sitemapURL.toNormalform(false), false);if (client.getStatusCode() != 200) {throw new IOException("Unable to download the sitemap file " + sitemapURL +diff --git a/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java b/source/net/yacy/document/parser/xml/opensearchdescriptionReader.javaindex 54bc8dbf9..37866ebc5 100644--- a/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java+++ b/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java@@ -147,19 +147,12 @@ public class opensearchdescriptionReader extends DefaultHandler {public opensearchdescriptionReader(final String path, final ClientIdentification.Agent agent) {this();this.agent = agent;- HTTPClient www = new HTTPClient(agent);- try {+ try (HTTPClient www = new HTTPClient(agent)) {www.GET(path, false);final SAXParser saxParser = getParser();saxParser.parse(www.getContentstream(), this);} catch (final Exception e) {ConcurrentLog.logException(e);- } finally {- try {- www.close();- } catch (final IOException e) {- ConcurrentLog.logException(e);- }}}@@ -170,8 +163,7 @@ public class opensearchdescriptionReader extends DefaultHandler {this.parsingTextValue = false;this.rssurl = null;this.atomurl = null;- HTTPClient www = new HTTPClient(this.agent);- try {+ try (HTTPClient www = new HTTPClient(this.agent)) {www.GET(path, false);final SAXParser saxParser = getParser();try {@@ -185,12 +177,6 @@ public class opensearchdescriptionReader extends DefaultHandler {} catch (final Exception e) {ConcurrentLog.warn("opensearchdescriptionReader", "parse exception: " + e);return false;- } finally {- try {- www.close();- } catch (final IOException e) {- ConcurrentLog.logException(e);- }}}diff --git a/source/net/yacy/http/ProxyHandler.java b/source/net/yacy/http/ProxyHandler.javaindex 5e1b482f2..2892c0039 100644--- a/source/net/yacy/http/ProxyHandler.java+++ b/source/net/yacy/http/ProxyHandler.java@@ -132,12 +132,11 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {RequestHeader proxyHeaders = ProxyHandler.convertHeaderFromJetty(request);setProxyHeaderForClient(request, proxyHeaders);- final HTTPClient client = new HTTPClient(ClientIdentification.yacyProxyAgent);- client.setTimout(timeout);- client.setHeader(proxyHeaders.entrySet());- client.setRedirecting(false);// send request- try {+ try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyProxyAgent)) {+ client.setTimout(timeout);+ client.setHeader(proxyHeaders.entrySet());+ client.setRedirecting(false);String queryString = request.getQueryString() != null ? "?" + request.getQueryString() : "";DigestURL digestURI = new DigestURL(request.getScheme(), request.getServerName(), request.getServerPort(), request.getRequestURI() + queryString);if (request.getMethod().equals(HeaderFramework.METHOD_GET)) {@@ -219,8 +218,6 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {}} catch (final SocketException se) {throw new ServletException("Socket Exception: " + se.getMessage());- } finally {- client.close();}// we handled this request, break out of handler chaindiff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.javaindex 4260da7bf..5611b033e 100644--- a/source/net/yacy/peers/Protocol.java+++ b/source/net/yacy/peers/Protocol.java@@ -159,11 +159,11 @@ public final class Protocol {final String path,final Map<String, ContentBody> parts,final int timeout) throws IOException {- final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);- httpClient.setTimout(timeout);- MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path);- this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false,- true);+ try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {+ httpClient.setTimout(timeout);+ MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path);+ this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false, true);+ }}/**@@ -197,19 +197,16 @@ public final class Protocol {final String salt = crypt.randomSalt();long responseTime = Long.MAX_VALUE;byte[] content = null;- try {+ try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000)) {// generate request- final Map<String, ContentBody> parts =- basicRequestParts(Switchboard.getSwitchboard(), null, salt);+ final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), null, salt);parts.put("count", UTF8.StringBody("20"));parts.put("magic", UTF8.StringBody(Long.toString(Network.magic)));parts.put("seed", UTF8.StringBody(mySeed.genSeedStr(salt)));// send requestfinal long start = System.currentTimeMillis();// final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), 30000, yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts);- final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000);- content =- httpClient.POSTbytes(+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/hello.html"),Seed.b64Hash2hexHash(targetHash) + ".yacyh",parts,@@ -433,41 +430,44 @@ public final class Protocol {parts.put("count", UTF8.StringBody(Integer.toString(maxCount)));parts.put("time", UTF8.StringBody(Long.toString(maxTime)));// final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts);- final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime);RSSReader reader = null;- for (final String ip: target.getIPs()) {- MultiProtocolURL targetBaseURL = null;- try {- targetBaseURL = target.getPublicMultiprotocolURL(ip, preferHttps);- byte[] result;- try {- result = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);- } catch(final IOException e) {- if(targetBaseURL.isHTTPS()) {- /* Failed with https : retry with http */- targetBaseURL = target.getPublicMultiprotocolURL(ip, false);- result = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);- if(result != null) {- /* Got something with http : mark peer SSL as unavailable on target peer */- markSSLUnavailableOnPeer(seedDB, target, ip, "yacyClient.queryRemoteCrawlURLs");- }- } else {- throw e;- }- }- reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);- } catch(MalformedURLException e) {- Network.log.warn("yacyClient.queryRemoteCrawlURLs malformed target URL for peer '" + target.getName()- + "' on address : " + ip);- } catch (final IOException e ) {- reader = null;- Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null");- }- if (reader != null) {- break;+ try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime)) {+ for (final String ip: target.getIPs()) {+ MultiProtocolURL targetBaseURL = null;+ try {+ targetBaseURL = target.getPublicMultiprotocolURL(ip, preferHttps);+ byte[] result;+ try {+ result = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);+ } catch(final IOException e) {+ if(targetBaseURL.isHTTPS()) {+ /* Failed with https : retry with http */+ targetBaseURL = target.getPublicMultiprotocolURL(ip, false);+ result = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);+ if(result != null) {+ /* Got something with http : mark peer SSL as unavailable on target peer */+ markSSLUnavailableOnPeer(seedDB, target, ip, "yacyClient.queryRemoteCrawlURLs");+ }+ } else {+ throw e;+ }+ }+ reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);+ } catch(MalformedURLException e) {+ Network.log.warn("yacyClient.queryRemoteCrawlURLs malformed target URL for peer '" + target.getName()+ + "' on address : " + ip);+ } catch (final IOException e ) {+ reader = null;+ Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null");+ }+ if (reader != null) {+ break;+ }+ target.put(Seed.RCOUNT, "0");+ seedDB.peerActions.interfaceDeparture(target, ip);}- target.put(Seed.RCOUNT, "0");- seedDB.peerActions.interfaceDeparture(target, ip);+ } catch (IOException e) {+ Network.log.warn(e);}final RSSFeed feed = reader == null ? null : reader.getFeed();@@ -962,13 +962,14 @@ public final class Protocol {//resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.crawlerUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/search.html"), 60000, target.getHexHash() + ".yacyh", parts));}- final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 8000);- byte[] a = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL + "/yacy/search.html"), hostname, parts, false, true);- if (a != null && a.length > 200000) {- // there is something wrong. This is too large, maybe a hack on the other side?- a = null;+ try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 8000)) {+ byte[] a = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL + "/yacy/search.html"), hostname, parts, false, true);+ if (a != null && a.length > 200000) {+ // there is something wrong. This is too large, maybe a hack on the other side?+ a = null;+ }+ resultMap = FileUtils.table(a);}- resultMap = FileUtils.table(a);// evaluate request resultif ( resultMap == null || resultMap.isEmpty() ) {@@ -1628,25 +1629,26 @@ public final class Protocol {}parts.put("lurlEntry", UTF8.StringBody(crypt.simpleEncode(lurlstr, salt)));// send request- final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 10000);- MultiProtocolURL targetBaseURL = target.getPublicMultiprotocolURL(ip, preferHttps);byte[] content;- try {- content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/crawlReceipt.html"),- target.getHexHash() + ".yacyh", parts, false, true);- } catch(final IOException e) {- if(targetBaseURL.isHTTPS()) {- /* Failed using https : retry with http */- targetBaseURL = target.getPublicMultiprotocolURL(ip, false);- content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/crawlReceipt.html"),- target.getHexHash() + ".yacyh", parts, false, true);- if(content != null) {- /* Success with http : mark SSL as unavailable on the target peer */- markSSLUnavailableOnPeer(sb.peers, target, ip, "yacyClient.crawlReceipt");- }- } else {- throw e;- }+ try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 10000)) {+ MultiProtocolURL targetBaseURL = target.getPublicMultiprotocolURL(ip, preferHttps);+ try {+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/crawlReceipt.html"),+ target.getHexHash() + ".yacyh", parts, false, true);+ } catch(final IOException e) {+ if(targetBaseURL.isHTTPS()) {+ /* Failed using https : retry with http */+ targetBaseURL = target.getPublicMultiprotocolURL(ip, false);+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/crawlReceipt.html"),+ target.getHexHash() + ".yacyh", parts, false, true);+ if(content != null) {+ /* Success with http : mark SSL as unavailable on the target peer */+ markSSLUnavailableOnPeer(sb.peers, target, ip, "yacyClient.crawlReceipt");+ }+ } else {+ throw e;+ }+ }}return FileUtils.table(content);} catch (final Exception e ) {@@ -1849,23 +1851,24 @@ public final class Protocol {parts.put("wordc", UTF8.StringBody(Integer.toString(indexes.size())));parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount)));parts.put("indexes", UTF8.StringBody(entrypost.toString()));- final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);byte[] content = null;- try {- content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),- targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);- } catch(final IOException e) {- if(targetBaseURL.isHTTPS()) {- targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);- /* Failed with https : retry with http on the same address */- content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),- targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);- if(content != null) {- /* Success with http : mark SSL as unavailable on the target peer */- markSSLUnavailableOnPeer(Switchboard.getSwitchboard().peers, targetSeed, ip, "yacyClient.transferRWI");- }- } else {- throw e;+ try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout)) {+ try {+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),+ targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);+ } catch(final IOException e) {+ if(targetBaseURL.isHTTPS()) {+ targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);+ /* Failed with https : retry with http on the same address */+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),+ targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);+ if(content != null) {+ /* Success with http : mark SSL as unavailable on the target peer */+ markSSLUnavailableOnPeer(Switchboard.getSwitchboard().peers, targetSeed, ip, "yacyClient.transferRWI");+ }+ } else {+ throw e;+ }}}final Iterator<String> v = FileUtils.strings(content);@@ -1953,20 +1956,21 @@ public final class Protocol {MultiProtocolURL targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps);parts.put("urlc", UTF8.StringBody(Integer.toString(urlc)));- final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);byte[] content = null;- try {- content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferURL.html"),- targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);- } catch(final IOException e) {- if(targetBaseURL.isHTTPS()) {- targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);- /* Failed with https : retry with http on the same address */+ try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout)) {+ try {content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferURL.html"),- targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);- } else {- throw e;- }+ targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);+ } catch(final IOException e) {+ if(targetBaseURL.isHTTPS()) {+ targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);+ /* Failed with https : retry with http on the same address */+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferURL.html"),+ targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);+ } else {+ throw e;+ }+ }}final Iterator<String> v = FileUtils.strings(content);@@ -1998,10 +2002,8 @@ public final class Protocol {SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);for (final String ip : targetSeed.getIPs()) {- try {- final Map<String, ContentBody> parts =- basicRequestParts(sb, targetSeed.hash, salt);- final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 15000);+ try (final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 15000)) {+ final Map<String, ContentBody> parts = basicRequestParts(sb, targetSeed.hash, salt);MultiProtocolURL targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps);byte[] content;try {diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.javaindex 8244a2e2a..ca0cd0a35 100644--- a/source/net/yacy/peers/SeedDB.java+++ b/source/net/yacy/peers/SeedDB.java@@ -897,19 +897,20 @@ public final class SeedDB implements AlternativeDomainNames {reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache, no-store"); // httpc uses HTTP/1.0 is this necessary?reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);- final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);- client.setHeader(reqHeader.entrySet());byte[] content = null;- try {- // send request- content = client.GETbytes(seedURL, null, null, false);- } catch (final Exception e) {- throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());- }-- // check response code- if (client.getHttpResponse().getStatusLine().getStatusCode() != 200) {- throw new IOException("Server returned status: " + client.getHttpResponse().getStatusLine());+ try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {+ client.setHeader(reqHeader.entrySet());+ try {+ // send request+ content = client.GETbytes(seedURL, null, null, false);+ } catch (final Exception e) {+ throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());+ }++ // check response code+ if (client.getHttpResponse().getStatusLine().getStatusCode() != 200) {+ throw new IOException("Server returned status: " + client.getHttpResponse().getStatusLine());+ }}try {@@ -1124,13 +1125,12 @@ public final class SeedDB implements AlternativeDomainNames {@Overridepublic void run() {// load the seed list- try {+ try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout)) {DigestURL url = new DigestURL(seedListFileURL);//final long start = System.currentTimeMillis();final RequestHeader reqHeader = new RequestHeader();reqHeader.put(HeaderFramework.PRAGMA, "no-cache");reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache, no-store");- final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);client.setHeader(reqHeader.entrySet());client.HEADResponse(url.toNormalform(false), false);diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.javaindex 2d7ab3ebd..493cd7a6f 100644--- a/source/net/yacy/server/http/HTTPDProxyHandler.java+++ b/source/net/yacy/server/http/HTTPDProxyHandler.java@@ -444,10 +444,10 @@ public final class HTTPDProxyHandler {requestHeader.remove(HeaderFramework.HOST);- final HTTPClient client = setupHttpClient(requestHeader, agent);-// send request- try {+ try (final HTTPClient client = new HTTPClient(agent, timeout)) {+ client.setHeader(requestHeader.entrySet());+ client.setRedirecting(false);client.GET(getUrl, false);if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());@@ -596,20 +596,7 @@ public final class HTTPDProxyHandler {}} // end hasBody} catch(final SocketException se) {- // if opened ...-// if(res != null) {-// // client cut proxy connection, abort download-// res.abort();-// }- client.close();handleProxyException(se,conProp,respond,url);- } finally {- // if opened ...-// if(res != null) {-// // ... close connection-// res.closeStream();-// }- client.close();}} catch (final Exception e) {handleProxyException(e,conProp,respond,url);@@ -759,20 +746,6 @@ public final class HTTPDProxyHandler {return domain;}- /**- * creates a new HttpClient and sets parameters according to proxy needs- *- * @param requestHeader- * @return- */- private static HTTPClient setupHttpClient(final RequestHeader requestHeader, final ClientIdentification.Agent agent) {- // setup HTTP-client- final HTTPClient client = new HTTPClient(agent, timeout);- client.setHeader(requestHeader.entrySet());- client.setRedirecting(false);- return client;- }-/*** determines in which form the response should be send and sets header accordingly* if the content length is not set we need to use chunked content encodingdiff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.javaindex 746a5900d..ab615e581 100644--- a/source/net/yacy/server/serverSwitch.java+++ b/source/net/yacy/server/serverSwitch.java@@ -686,23 +686,13 @@ public class serverSwitch {final String[] uris = CommonPattern.COMMA.split(uri);for (String netdef : uris) {netdef = netdef.trim();- try {+ try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {final RequestHeader reqHeader = new RequestHeader();- reqHeader- .put(HeaderFramework.USER_AGENT,- ClientIdentification.yacyInternetCrawlerAgent.userAgent);- final HTTPClient client = new HTTPClient(- ClientIdentification.yacyInternetCrawlerAgent);+ reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);client.setHeader(reqHeader.entrySet());- byte[] data = client- .GETbytes(- uri,- getConfig(- SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,- "admin"),- getConfig(- SwitchboardConstants.ADMIN_ACCOUNT_B64MD5,- ""), false);+ byte[] data = client.GETbytes(uri,+ getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),+ getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), false);if (data == null || data.length == 0) {continue;}diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.javaindex 89499ef02..4ed9b59ce 100644--- a/source/net/yacy/yacy.java+++ b/source/net/yacy/yacy.java@@ -532,9 +532,7 @@ public final class yacy {final String adminUser = config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");// send 'wget' to web interface- final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);- // con.setHeader(requestHeader.entrySet());- try {+ try (final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {/* First get a valid transaction token using HTTP GET */con.GETbytes("http://localhost:"+ port +"/" + path, adminUser, encodedPassword, false);@@ -600,9 +598,7 @@ public final class yacy {if (encodedPassword == null) encodedPassword = ""; // not defined// send 'wget' to web interface- final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);- // con.setHeader(requestHeader.entrySet());- try {+ try (final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {con.GETbytes("http://localhost:"+ port +"/" + path, config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword, false);if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);