Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
implemented (most of) https://github.com/yacy/yacy_search_server/issues/727 some refactorings done as well to support multiple user agents instead of only one; per request then is one of those user agents selected, not one random per session. Also: made sure that no robots.txt is loaded any more in case that the user agent is not a robot user agent.
Michael Peter Christenacb77bd68ab0
diff --git a/source/net/yacy/cora/protocol/ClientIdentification.java b/source/net/yacy/cora/protocol/ClientIdentification.javaindex 8e3eba44b..25382f040 100644--- a/source/net/yacy/cora/protocol/ClientIdentification.java+++ b/source/net/yacy/cora/protocol/ClientIdentification.java@@ -31,40 +31,55 @@ import java.util.concurrent.ConcurrentHashMap;public class ClientIdentification {- public static final int clientTimeoutInit = 10000;+ public static final int clientTimeoutInit = 30000;public static final int minimumLocalDeltaInit = 10; // the minimum time difference between access of the same local domainpublic static final int minimumGlobalDeltaInit = 250; // the minimum time difference between access of the same global domainpublic static class Agent {- public final String userAgent; // the name that is send in http request to identify the agent- public final String[] robotIDs; // the name that is used in robots.txt to identify the agent- public final int minimumDelta; // the minimum delay between two accesses- public final int clientTimeout;- public Agent(final String userAgent, final String[] robotIDs, final int minimumDelta, final int clientTimeout) {- this.userAgent = userAgent;+ private final String[] userAgents; // the name that is send in http request to identify the agent+ private final String[] robotIDs; // the name that is used in robots.txt to identify the agent; might be null or empty to signal that no robots.txt must be loaded (should only be use for browser agents)+ private final int minimumDelta; // the minimum delay between two accesses+ private final int clientTimeout;++ public Agent(final String[] userAgents, final String[] robotIDs, final int minimumDelta, final int clientTimeout) {+ this.userAgents = userAgents;this.robotIDs = robotIDs;this.minimumDelta = minimumDelta;this.clientTimeout = clientTimeout;}++ public String userAgent() {+ return this.userAgents.length == 1 ? this.userAgents[0] : this.userAgents[random.nextInt(this.userAgents.length)];+ }++ public boolean isRobot() {+ return this.robotIDs != null && this.robotIDs.length > 0;+ }++ public String[] robotIDs() {+ return this.robotIDs == null || this.robotIDs.length == 0 ? new String[] {"Mozilla"} : this.robotIDs;+ }++ public int minimumDelta() {+ return this.minimumDelta;+ }++ public int clientTimeout() {+ return this.clientTimeout;+ }}- private final static String[] browserAgents = new String[]{ // fake browser user agents are NOT AVAILABLE IN P2P OPERATION, only on special customer configurations (commercial users demanded this, I personally think this is inadvisable)- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36",- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:22.0) Gecko/20100101 Firefox/22.0",- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36",- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36",- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/536.30.1 (KHTML, like Gecko) Version/6.0.5 Safari/536.30.1",- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36",- "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36",- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36",- "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0",- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0",- "Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/20100101 Firefox/22.0",- "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20100101 Firefox/22.0",- "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36",- "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0"- };+ private final static String[] browserAgents = new String[]{+ // see https://github.com/yacy/yacy_search_server/issues/727+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",+ "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.3",+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3",+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.3",+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0",+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36"+ };private static final Random random = new Random(System.currentTimeMillis());private static Map<String, Agent> agents = new ConcurrentHashMap<>();public final static String yacyInternetCrawlerAgentName = "YaCy Internet (cautious)";@@ -72,9 +87,10 @@ public class ClientIdentification {public final static String yacyIntranetCrawlerAgentName = "YaCy Intranet (greedy)";public static Agent yacyIntranetCrawlerAgent = null; // defined later in staticpublic final static String googleAgentName = "Googlebot";- public final static Agent googleAgentAgent = new Agent("Googlebot/2.1 (+http://www.google.com/bot.html)", new String[]{"Googlebot", "Googlebot-Mobile"}, minimumGlobalDeltaInit / 10, clientTimeoutInit);+ //public final static Agent googleAgentAgent = new Agent("Googlebot/2.1 (+http://www.google.com/bot.html)", new String[]{"Googlebot", "Googlebot-Mobile"}, minimumGlobalDeltaInit / 10, clientTimeoutInit);+ public final static Agent googleAgentAgent = new Agent(new String[]{"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}, new String[]{"Googlebot", "Googlebot-Mobile"}, minimumGlobalDeltaInit / 10, clientTimeoutInit); // see https://github.com/yacy/yacy_search_server/issues/727public final static String yacyProxyAgentName = "YaCyProxy";- public final static Agent yacyProxyAgent = new Agent("yacy - this is a proxy access through YaCy from a browser, not a robot (the yacy bot user agent is 'yacybot')", new String[]{"yacy"}, minimumGlobalDeltaInit, clientTimeoutInit);+ public final static Agent yacyProxyAgent = new Agent(new String[]{"yacy - this is a proxy access through YaCy from a browser, not a robot (the yacy bot user agent is 'yacybot')"}, new String[]{"yacy"}, minimumGlobalDeltaInit, clientTimeoutInit);public final static String customAgentName = "Custom Agent";public final static String browserAgentName = "Random Browser";public static Agent browserAgent;@@ -88,7 +104,7 @@ public class ClientIdentification {static {generateYaCyBot("new");- browserAgent = new Agent(browserAgents[random.nextInt(browserAgents.length)], new String[]{"Mozilla"}, minimumLocalDeltaInit, clientTimeoutInit);+ browserAgent = new Agent(browserAgents, new String[0], minimumGlobalDeltaInit, clientTimeoutInit);agents.put(googleAgentName, googleAgentAgent);agents.put(browserAgentName, browserAgent);agents.put(yacyProxyAgentName, yacyProxyAgent);@@ -101,8 +117,8 @@ public class ClientIdentification {*/public static void generateYaCyBot(final String addinfo) {final String agentString = "yacybot (" + addinfo + "; " + yacySystem + ") https://yacy.net/bot.html";- yacyInternetCrawlerAgent = new Agent(agentString, new String[]{"yacybot"}, minimumGlobalDeltaInit, clientTimeoutInit);- yacyIntranetCrawlerAgent = new Agent(agentString, new String[]{"yacybot"}, minimumLocalDeltaInit, clientTimeoutInit); // must have the same userAgent String as the web crawler because this is also used for snippets+ yacyInternetCrawlerAgent = new Agent(new String[]{agentString}, new String[]{"yacybot"}, minimumGlobalDeltaInit, clientTimeoutInit);+ yacyIntranetCrawlerAgent = new Agent(new String[]{agentString}, new String[]{"yacybot"}, minimumLocalDeltaInit, clientTimeoutInit); // must have the same userAgent String as the web crawler because this is also used for snippetsagents.put(yacyInternetCrawlerAgentName, yacyInternetCrawlerAgent);agents.put(yacyIntranetCrawlerAgentName, yacyIntranetCrawlerAgent);}@@ -110,7 +126,7 @@ public class ClientIdentification {public static void generateCustomBot(final String name, final String string, final int minimumdelta, final int clienttimeout) {if (name.toLowerCase().indexOf("yacy") >= 0 || string.toLowerCase().indexOf("yacy") >= 0) return; // don't allow 'yacy' in custom bot stringsfinal String agentString = string.replace("$$SYSTEM$$", yacySystem.replace("java", "O"));- agents.put(customAgentName, new Agent(agentString, new String[]{name}, minimumdelta, clienttimeout));+ agents.put(customAgentName, new Agent(new String[]{agentString}, new String[]{name}, minimumdelta, clienttimeout));}/**diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.javaindex 015a13c36..3ff85a567 100644--- a/source/net/yacy/cora/protocol/http/HTTPClient.java+++ b/source/net/yacy/cora/protocol/http/HTTPClient.java@@ -181,18 +181,18 @@ public class HTTPClient implements Closeable {public HTTPClient(final ClientIdentification.Agent agent) {super();- this.timeout = agent.clientTimeout;+ this.timeout = agent.clientTimeout();this.clientBuilder = initClientBuilder();- this.clientBuilder.setUserAgent(agent.userAgent);+ this.clientBuilder.setUserAgent(agent.userAgent());this.reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG);- this.setTimout(agent.clientTimeout);+ this.setTimout(agent.clientTimeout());}public HTTPClient(final ClientIdentification.Agent agent, final int timeout) {super();this.timeout = timeout;this.clientBuilder = initClientBuilder();- this.clientBuilder.setUserAgent(agent.userAgent);+ this.clientBuilder.setUserAgent(agent.userAgent());this.reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG);this.setTimout(timeout);}@@ -222,7 +222,7 @@ public class HTTPClient implements Closeable {builder.setDefaultRequestConfig(DFLTREQUESTCONFIG);// UserAgent- builder.setUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);+ builder.setUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent());// remove retries; we expect connections to fail; therefore we should not retry//builder.disableAutomaticRetries();@@ -360,7 +360,7 @@ public class HTTPClient implements Closeable {* @param userAgent*/public void setUserAgent(final ClientIdentification.Agent agent) {- this.clientBuilder.setUserAgent(agent.userAgent);+ this.clientBuilder.setUserAgent(agent.userAgent());}/**diff --git a/source/net/yacy/cora/util/Html2Image.java b/source/net/yacy/cora/util/Html2Image.javaindex 83efe559b..ccd44d31a 100644--- a/source/net/yacy/cora/util/Html2Image.java+++ b/source/net/yacy/cora/util/Html2Image.java@@ -26,8 +26,6 @@ import java.awt.Graphics;import java.awt.Image;import java.awt.MediaTracker;import java.awt.image.BufferedImage;-import java.beans.PropertyChangeEvent;-import java.beans.PropertyChangeListener;import java.io.File;import java.io.IOException;import java.util.List;@@ -463,10 +461,7 @@ public class Html2Image {};htmlPane.setEditorKitForContentType("text/html", kit);htmlPane.setContentType("text/html");- htmlPane.addPropertyChangeListener(new PropertyChangeListener() {- @Override- public void propertyChange(final PropertyChangeEvent evt) {- }+ htmlPane.addPropertyChangeListener(evt -> {});// load the page@@ -524,7 +519,7 @@ public class Html2Image {exitStatus = 1;return;}- if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,+ if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent(),"en-us,en;q=0.5", targetPdfFile, 30)) {if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {diff --git a/source/net/yacy/crawler/HostBalancer.java b/source/net/yacy/crawler/HostBalancer.javaindex c9b8133cf..afec24e3e 100644--- a/source/net/yacy/crawler/HostBalancer.java+++ b/source/net/yacy/crawler/HostBalancer.java@@ -67,7 +67,7 @@ public class HostBalancer implements Balancer {private final static ConcurrentLog log = new ConcurrentLog("HostBalancer");public final static HandleMap depthCache = new RowHandleMap(Word.commonHashLength, Word.commonHashOrder, 2, 8 * 1024 * 1024, "HostBalancer.DepthCache");-+ public final static ClientIdentification.Agent unknwonAgentDefault = ClientIdentification.browserAgent;private final File hostsPath;private final boolean exceed134217727;private final ConcurrentHashMap<String, HostQueue> queues;@@ -347,7 +347,8 @@ public class HostBalancer implements Balancer {final String hosthash = i.next();final HostQueue hq = this.queues.get(hosthash);if (hq == null) {removehosts.add(hosthash); i.remove(); continue smallstacks;}- final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hosthash, robots, ClientIdentification.yacyInternetCrawlerAgent);++ final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hosthash, robots, unknwonAgentDefault);if (delta == Integer.MIN_VALUE) {// never-crawled hosts; we do not want to have too many of them in here. Loading new hosts means: waiting for robots.txt to loadfreshhosts.add(hosthash);@@ -404,7 +405,7 @@ public class HostBalancer implements Balancer {mixedstrategy: for (final String h: this.roundRobinHostHashes) {final HostQueue hq = this.queues.get(h);if (hq != null) {- int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), h, robots, ClientIdentification.yacyInternetCrawlerAgent) / 200;+ int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), h, robots, unknwonAgentDefault) / 200;if (delta < 0) delta = 0;List<String> queueHashes = fastTree.get(delta);if (queueHashes == null) {@@ -499,7 +500,7 @@ public class HostBalancer implements Balancer {final String s = i.next();final HostQueue hq = this.queues.get(s);if (hq == null) {i.remove(); continue protectcheck;}- final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), s, robots, ClientIdentification.yacyInternetCrawlerAgent);+ final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), s, robots, unknwonAgentDefault);if (delta >= 0) {i.remove();}}}@@ -529,7 +530,7 @@ public class HostBalancer implements Balancer {@SuppressWarnings("unchecked")final Iterator<Request>[] hostIterator = (Iterator<Request>[]) Array.newInstance(Iterator.class, 1);hostIterator[0] = null;- return new Iterator<Request>() {+ return new Iterator<>() {@Overridepublic boolean hasNext() {return hostsIterator.hasNext() || (hostIterator[0] != null && hostIterator[0].hasNext());@@ -560,7 +561,7 @@ public class HostBalancer implements Balancer {public Map<String, Integer[]> getDomainStackHosts(final RobotsTxt robots) {final Map<String, Integer[]> map = new TreeMap<>(); // we use a tree map to get a stable orderingfor (final HostQueue hq: this.queues.values()) {- final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hq.getHostHash(), robots, ClientIdentification.yacyInternetCrawlerAgent);+ final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hq.getHostHash(), robots, unknwonAgentDefault);map.put(hq.getHost() + ":" + hq.getPort(), new Integer[]{hq.size(), delta});}return map;diff --git a/source/net/yacy/crawler/data/CrawlQueues.java b/source/net/yacy/crawler/data/CrawlQueues.javaindex 38c8e8a2c..fd0fd971e 100644--- a/source/net/yacy/crawler/data/CrawlQueues.java+++ b/source/net/yacy/crawler/data/CrawlQueues.java@@ -72,7 +72,7 @@ import net.yacy.search.index.ErrorCache;import net.yacy.search.index.ErrorCacheFiller;public class CrawlQueues {-+private final static Request POISON_REQUEST = new Request();private final static ConcurrentLog log = new ConcurrentLog("CRAWLER");@@ -83,7 +83,7 @@ public class CrawlQueues {public NoticedURL noticeURL;public ErrorCache errorURL;-+/** URLs pulled by remote peers in order to crawl them for us */public Map<String, DigestURL> delegatedURL;@@ -91,9 +91,9 @@ public class CrawlQueues {this.sb = sb;final int maxWorkers = (int) sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10);this.worker = new Loader[maxWorkers];- /* We initialize workerQueue with the same capacity as worker array, because this same queue+ /* We initialize workerQueue with the same capacity as worker array, because this same queue* will be used to send POISON_REQUEST items consumed by all eventually running workers in the close() function*/- this.workerQueue = new ArrayBlockingQueue<Request>(maxWorkers);+ this.workerQueue = new ArrayBlockingQueue<>(maxWorkers);this.remoteCrawlProviderHashes = null;// start crawling management@@ -107,9 +107,9 @@ public class CrawlQueues {}public void initRemoteCrawlQueues () {- if (this.remoteCrawlProviderHashes == null) this.remoteCrawlProviderHashes = new ArrayList<String>();+ if (this.remoteCrawlProviderHashes == null) this.remoteCrawlProviderHashes = new ArrayList<>();if (this.delegatedURL == null) {- this.delegatedURL = new ConcurrentHashMap<String, DigestURL>();+ this.delegatedURL = new ConcurrentHashMap<>();log.config("Finished Startup of Crawling Management");}}@@ -125,10 +125,10 @@ public class CrawlQueues {this.errorURL.clearCache();/* Concurrently refill the error cache with recent errors from the index */new ErrorCacheFiller(this.sb, this.errorURL).start();-+if (this.remoteCrawlProviderHashes != null) this.remoteCrawlProviderHashes.clear();this.noticeURL.close();- this.noticeURL = new NoticedURL(newQueuePath, sb.getConfigInt("crawler.onDemandLimit", 1000), this.sb.exceed134217727);+ this.noticeURL = new NoticedURL(newQueuePath, this.sb.getConfigInt("crawler.onDemandLimit", 1000), this.sb.exceed134217727);if (this.delegatedURL != null) this.delegatedURL.clear();}@@ -139,7 +139,7 @@ public class CrawlQueues {this.workerQueue.clear();// wait for all workers to finishfor (int i = 0; i < this.workerQueue.remainingCapacity(); i++) {- /* We use the non-blocking offer() function instead of the blocking put() in the unlikely eventual case another thread+ /* We use the non-blocking offer() function instead of the blocking put() in the unlikely eventual case another thread* added an element to workerQueue during this loop.*/try {this.workerQueue.offer(POISON_REQUEST, 1, TimeUnit.SECONDS);@@ -181,14 +181,14 @@ public class CrawlQueues {//if (this.noticeURL.existsInStack(hash)) {// return HarvestProcess.CRAWLER;//} // this is disabled because it prevents proper crawling of smb shares. The cause is unknown- for (final Request request: activeWorkerEntries().values()) {+ for (final Request request: this.activeWorkerEntries().values()) {if (Base64Order.enhancedCoder.equal(request.url().hash(), hash)) {return HarvestProcess.WORKER;}}return null;}-+/*** count the number of same host names in the worker* @param host@@ -197,7 +197,7 @@ public class CrawlQueues {public int hostcount(final String host) {if (host == null || host.length() == 0) return 0;int c = 0;- for (final DigestURL url: activeWorkerEntries().keySet()) {+ for (final DigestURL url: this.activeWorkerEntries().keySet()) {if (host.equals(url.getHost())) {c++;}@@ -212,12 +212,12 @@ public class CrawlQueues {this.delegatedURL.remove(ASCII.String(hash));}}-+public int removeHosts(final Set<String> hosthashes) {return this.noticeURL.removeByHostHash(hosthashes);//this.delegatedURL.remove(hash);}-+public DigestURL getURL(final byte[] urlhash) {assert urlhash != null;if (urlhash == null || urlhash.length == 0) {@@ -229,7 +229,7 @@ public class CrawlQueues {return u;}}- for (final DigestURL url: activeWorkerEntries().keySet()) {+ for (final DigestURL url: this.activeWorkerEntries().keySet()) {if (Base64Order.enhancedCoder.equal(url.hash(), urlhash)) {return url;}@@ -249,10 +249,10 @@ public class CrawlQueues {this.errorURL.clearStack();}}-+public Map<DigestURL, Request> activeWorkerEntries() {synchronized (this.worker) {- Map<DigestURL, Request> map = new HashMap<DigestURL, Request>();+ Map<DigestURL, Request> map = new HashMap<>();for (final Loader w: this.worker) {if (w != null) {Request r = w.loading();@@ -271,19 +271,19 @@ public class CrawlQueues {final boolean robinsonPrivateCase = (this.sb.isRobinsonMode() &&!this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER));- if ((robinsonPrivateCase || coreCrawlJobSize() <= 20) && limitCrawlJobSize() > 0) {+ if ((robinsonPrivateCase || this.coreCrawlJobSize() <= 20) && this.limitCrawlJobSize() > 0) {// move some tasks to the core crawl job so we have something to do- final int toshift = Math.min(10, limitCrawlJobSize()); // this cannot be a big number because the balancer makes a forced waiting if it cannot balance+ final int toshift = Math.min(10, this.limitCrawlJobSize()); // this cannot be a big number because the balancer makes a forced waiting if it cannot balancefor (int i = 0; i < toshift; i++) {this.noticeURL.shift(NoticedURL.StackType.GLOBAL, NoticedURL.StackType.LOCAL, this.sb.crawler, this.sb.robots);}- CrawlQueues.log.info("shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()=" + coreCrawlJobSize() +- ", limitCrawlJobSize()=" + limitCrawlJobSize() + ", cluster.mode=" + this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "") ++ CrawlQueues.log.info("shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()=" + this.coreCrawlJobSize() ++ ", limitCrawlJobSize()=" + this.limitCrawlJobSize() + ", cluster.mode=" + this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "") +", robinsonMode=" + ((this.sb.isRobinsonMode()) ? "on" : "off"));}- final String queueCheckCore = loadIsPossible(NoticedURL.StackType.LOCAL);- final String queueCheckNoload = loadIsPossible(NoticedURL.StackType.NOLOAD);+ final String queueCheckCore = this.loadIsPossible(NoticedURL.StackType.LOCAL);+ final String queueCheckNoload = this.loadIsPossible(NoticedURL.StackType.NOLOAD);if (queueCheckCore != null && queueCheckNoload != null) {if (CrawlQueues.log.isFine()) {CrawlQueues.log.fine("omitting de-queue/local: " + queueCheckCore + ":" + queueCheckNoload);@@ -291,7 +291,7 @@ public class CrawlQueues {return false;}- if (isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {+ if (this.isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {if (CrawlQueues.log.isFine()) {CrawlQueues.log.fine("omitting de-queue/local: paused");}@@ -304,7 +304,7 @@ public class CrawlQueues {final String stats = "LOCALCRAWL[" +this.noticeURL.stackSize(NoticedURL.StackType.NOLOAD) + ", " +this.noticeURL.stackSize(NoticedURL.StackType.LOCAL) + ", " +- this.noticeURL.stackSize(NoticedURL.StackType.GLOBAL) ++ this.noticeURL.stackSize(NoticedURL.StackType.GLOBAL) +", " + this.noticeURL.stackSize(NoticedURL.StackType.REMOTE) + "]";try {if (!this.noticeURL.isEmpty(NoticedURL.StackType.NOLOAD)) {@@ -338,7 +338,7 @@ public class CrawlQueues {CrawlQueues.log.severe(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());return true;}- load(urlEntry, stats);+ this.load(urlEntry, stats);return true;} catch (final IOException e) {CrawlQueues.log.severe(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);@@ -381,9 +381,9 @@ public class CrawlQueues {if (urlEntry == null || urlEntry.url() == null) {CrawlQueues.log.info(stats + ": urlEntry = null");} else {- if (!activeWorkerEntries().containsKey(urlEntry.url())) {+ if (!this.activeWorkerEntries().containsKey(urlEntry.url())) {try {- ensureLoaderRunning();+ this.ensureLoaderRunning();this.workerQueue.put(urlEntry);} catch (InterruptedException e) {ConcurrentLog.logException(e);@@ -471,14 +471,14 @@ public class CrawlQueues {return false;}- if (remoteTriggeredCrawlJobSize() > 200) {+ if (this.remoteTriggeredCrawlJobSize() > 200) {if (CrawlQueues.log.isFine()) {CrawlQueues.log.fine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing");}return false;}- if (coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) {+ if (this.coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) {if (CrawlQueues.log.isFine()) {CrawlQueues.log.fine("remoteCrawlLoaderJob: a local crawl is running, omitting processing");}@@ -525,12 +525,12 @@ public class CrawlQueues {}// we know a peer which should provide remote crawl entries. load them now.- final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,+ final boolean preferHttps = this.sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);final RSSFeed feed = Protocol.queryRemoteCrawlURLs(this.sb.peers, seed, 60, 10000, preferHttps);if (feed == null || feed.isEmpty()) {// try again and ask another peer- return remoteCrawlLoaderJob();+ return this.remoteCrawlLoaderJob();}// parse the rss@@ -573,24 +573,24 @@ public class CrawlQueues {}return true;}-+public boolean autocrawlJob() {if (!this.sb.getConfigBool(SwitchboardConstants.AUTOCRAWL, false)) {return false;}-- if (isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {++ if (this.isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {return false;}-- if (coreCrawlJobSize() > 200) {++ if (this.coreCrawlJobSize() > 200) {return false;}-+String rows = this.sb.getConfig(SwitchboardConstants.AUTOCRAWL_ROWS, "100");-+String dateQuery = String.format("load_date_dt:[* TO NOW-%sDAY]", this.sb.getConfig(SwitchboardConstants.AUTOCRAWL_DAYS, "1"));-+final SolrQuery query = new SolrQuery();query.add("group", "true");query.add("group.field", "host_s");@@ -601,10 +601,10 @@ public class CrawlQueues {query.setFields("host_s,url_protocol_s");query.addSort("load_date_dt", SolrQuery.ORDER.asc);query.addFilterQuery(dateQuery);-+try {- QueryResponse resp = sb.index.fulltext().getDefaultConnector().getResponseByParams(query);-+ QueryResponse resp = this.sb.index.fulltext().getDefaultConnector().getResponseByParams(query);+int i = 0;int deepRatio = Integer.parseInt(this.sb.getConfig(SwitchboardConstants.AUTOCRAWL_RATIO, "50"));for (SolrDocument doc: resp.getResults()) {@@ -619,7 +619,7 @@ public class CrawlQueues {DigestURL url;if (doc.getFieldValue("url_protocol_s") == null || doc.getFieldValue("host_s") == null) {//Skip this document if either of these values is null.- continue;+ continue;}final String u = doc.getFieldValue("url_protocol_s").toString() + "://" + doc.getFieldValue("host_s").toString();try {@@ -643,13 +643,13 @@ public class CrawlQueues {CrawlQueues.log.warn("autocrawl: Rejected URL '" + urlToString(url) + "': " + urlRejectReason);}}-+} catch (SolrException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();}-+return true;}@@ -678,7 +678,7 @@ public class CrawlQueues {// do nothing if either there are private processes to be done// or there is no global crawl on the stack- final String queueCheck = loadIsPossible(NoticedURL.StackType.REMOTE);+ final String queueCheck = this.loadIsPossible(NoticedURL.StackType.REMOTE);if (queueCheck != null) {if (CrawlQueues.log.isFinest()) {CrawlQueues.log.finest("omitting de-queue/remote: " + queueCheck);@@ -686,7 +686,7 @@ public class CrawlQueues {return false;}- if (isPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {+ if (this.isPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {if (CrawlQueues.log.isFinest()) {CrawlQueues.log.finest("omitting de-queue/remote: paused");}@@ -699,7 +699,7 @@ public class CrawlQueues {try {final Request urlEntry = this.noticeURL.pop(NoticedURL.StackType.REMOTE, true, this.sb.crawler, this.sb.robots);if (urlEntry == null) return false;- load(urlEntry, stats);+ this.load(urlEntry, stats);return true;} catch (final IOException e) {CrawlQueues.log.severe(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);@@ -721,84 +721,85 @@ public class CrawlQueues {if (this.worker[i].loading() == null) return;}}-+private final class Loader extends Thread {private Request request = null;private Loader() {}-+public Request loading() {- return request;+ return this.request;}@Overridepublic void run() {this.setPriority(Thread.MIN_PRIORITY); // http requests from the crawler should not cause that other functions work worsetry {- while ((request = CrawlQueues.this.workerQueue.poll(10, TimeUnit.SECONDS)) != POISON_REQUEST) {- if (request == null) break; // we run this only for a specific time and then let the process die to clear up resources- request.setStatus("worker-initialized", WorkflowJob.STATUS_INITIATED);- this.setName("CrawlQueues.Loader(" + request.url().toNormalform(false) + ")");- CrawlProfile profile = CrawlQueues.this.sb.crawler.get(UTF8.getBytes(request.profileHandle()));+ while ((this.request = CrawlQueues.this.workerQueue.poll(10, TimeUnit.SECONDS)) != POISON_REQUEST) {+ if (this.request == null) break; // we run this only for a specific time and then let the process die to clear up resources+ this.request.setStatus("worker-initialized", WorkflowJob.STATUS_INITIATED);+ this.setName("CrawlQueues.Loader(" + this.request.url().toNormalform(false) + ")");+ CrawlProfile profile = CrawlQueues.this.sb.crawler.get(UTF8.getBytes(this.request.profileHandle()));try {// checking robots.txt for http(s) resources- request.setStatus("worker-checkingrobots", WorkflowJob.STATUS_STARTED);+ this.request.setStatus("worker-checkingrobots", WorkflowJob.STATUS_STARTED);RobotsTxtEntry robotsEntry;- if ((request.url().getProtocol().equals("http") || request.url().getProtocol().equals("https")) &&- (robotsEntry = CrawlQueues.this.sb.robots.getEntry(request.url(), profile.getAgent())) != null &&- robotsEntry.isDisallowed(request.url())) {+ if ((this.request.url().getProtocol().equals("http") || this.request.url().getProtocol().equals("https")) &&+ profile.getAgent().isRobot() &&+ (robotsEntry = CrawlQueues.this.sb.robots.getEntry(this.request.url(), profile.getAgent())) != null &&+ robotsEntry.isDisallowed(this.request.url())) {//if (log.isFine()) log.logFine("Crawling of URL '" + request.url().toString() + "' disallowed by robots.txt.");- CrawlQueues.this.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_ROBOTS_RULE, "denied by robots.txt", -1);- request.setStatus("worker-disallowed", WorkflowJob.STATUS_FINISHED);+ CrawlQueues.this.errorURL.push(this.request.url(), this.request.depth(), profile, FailCategory.FINAL_ROBOTS_RULE, "denied by robots.txt", -1);+ this.request.setStatus("worker-disallowed", WorkflowJob.STATUS_FINISHED);} else {// starting a load from the internet- request.setStatus("worker-loading", WorkflowJob.STATUS_RUNNING);+ this.request.setStatus("worker-loading", WorkflowJob.STATUS_RUNNING);String error = null;-+// load a resource and push queue entry to switchboard queue// returns null if everything went fine, a fail reason string if a problem occurredtry {- request.setStatus("loading", WorkflowJob.STATUS_RUNNING);- final Response response = CrawlQueues.this.sb.loader.load(request, profile == null ? CacheStrategy.IFEXIST : profile.cacheStrategy(), BlacklistType.CRAWLER, profile.getAgent());+ this.request.setStatus("loading", WorkflowJob.STATUS_RUNNING);+ final Response response = CrawlQueues.this.sb.loader.load(this.request, profile == null ? CacheStrategy.IFEXIST : profile.cacheStrategy(), BlacklistType.CRAWLER, profile.getAgent());if (response == null) {- request.setStatus("error", WorkflowJob.STATUS_FINISHED);+ this.request.setStatus("error", WorkflowJob.STATUS_FINISHED);if (CrawlQueues.log.isFine()) {- CrawlQueues.log.fine("problem loading " + request.url().toString() + ": no content (possibly caused by cache policy)");+ CrawlQueues.log.fine("problem loading " + this.request.url().toString() + ": no content (possibly caused by cache policy)");}error = "no content (possibly caused by cache policy)";} else {- request.setStatus("loaded", WorkflowJob.STATUS_RUNNING);+ this.request.setStatus("loaded", WorkflowJob.STATUS_RUNNING);final String storedFailMessage = CrawlQueues.this.sb.toIndexer(response);- request.setStatus("enqueued-" + ((storedFailMessage == null) ? "ok" : "fail"), WorkflowJob.STATUS_FINISHED);+ this.request.setStatus("enqueued-" + ((storedFailMessage == null) ? "ok" : "fail"), WorkflowJob.STATUS_FINISHED);error = (storedFailMessage == null) ? null : "not enqueued to indexer: " + storedFailMessage;}} catch (final IOException e) {- request.setStatus("error", WorkflowJob.STATUS_FINISHED);+ this.request.setStatus("error", WorkflowJob.STATUS_FINISHED);if (CrawlQueues.log.isFine()) {- CrawlQueues.log.fine("problem loading " + request.url().toString() + ": " + e.getMessage());+ CrawlQueues.log.fine("problem loading " + this.request.url().toString() + ": " + e.getMessage());}error = "load error - " + e.getMessage();}-+if (error != null) {if (error.endsWith("$")) {// the "$" mark at the end of the error message means, that the error was already pushed to the error-db by the reporting method// thus we only push this message if we don't have that markerror = error.substring(0, error.length() - 1).trim();} else {- CrawlQueues.this.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "cannot load: " + error, -1);+ CrawlQueues.this.errorURL.push(this.request.url(), this.request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "cannot load: " + error, -1);}- request.setStatus("worker-error", WorkflowJob.STATUS_FINISHED);+ this.request.setStatus("worker-error", WorkflowJob.STATUS_FINISHED);} else {- request.setStatus("worker-processed", WorkflowJob.STATUS_FINISHED);+ this.request.setStatus("worker-processed", WorkflowJob.STATUS_FINISHED);}}} catch (final Exception e) {- CrawlQueues.this.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, e.getMessage() + " - in worker", -1);- request.setStatus("worker-exception", WorkflowJob.STATUS_FINISHED);+ CrawlQueues.this.errorURL.push(this.request.url(), this.request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, e.getMessage() + " - in worker", -1);+ this.request.setStatus("worker-exception", WorkflowJob.STATUS_FINISHED);} finally {- request = null;+ this.request = null;this.setName("CrawlQueues.Loader(WAITING)");}profile = null;diff --git a/source/net/yacy/crawler/data/Latency.java b/source/net/yacy/crawler/data/Latency.javaindex 07c3faba2..8aab14514 100644--- a/source/net/yacy/crawler/data/Latency.java+++ b/source/net/yacy/crawler/data/Latency.java@@ -122,6 +122,7 @@ public class Latency {* @return the waiting time in milliseconds; 0 if not known; -1 if host gives us special rights*/public static int waitingRobots(final MultiProtocolURL url, final RobotsTxt robots, final ClientIdentification.Agent agent) {+ if (!agent.isRobot()) return agent.minimumDelta();int robotsDelay;final RobotsTxtEntry robotsEntry = robots.getEntry(url, agent);robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis();@@ -130,6 +131,7 @@ public class Latency {}private static int waitingRobots(final String hostport, final RobotsTxt robots, final ClientIdentification.Agent agent, final boolean fetchOnlineIfNotAvailableOrNotFresh) {+ if (!agent.isRobot()) return agent.minimumDelta();int robotsDelay;final RobotsTxtEntry robotsEntry = robots.getEntry(hostport, agent, fetchOnlineIfNotAvailableOrNotFresh);robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis();@@ -154,7 +156,7 @@ public class Latency {if (host == null) return Integer.MIN_VALUE; // no delay if host is new; use Integer because there is a cast to int somewhere// find the minimum waiting time based on the network domain (local or global)- int waiting = agent.minimumDelta;+ int waiting = agent.minimumDelta();// if we have accessed the domain many times, get slower (the flux factor)waiting += host.flux(waiting);@@ -199,7 +201,7 @@ public class Latency {// find the minimum waiting time based on the network domain (local or global)final boolean local = url.isLocal();- int waiting = agent.minimumDelta;+ int waiting = agent.minimumDelta();// if we have accessed the domain many times, get slower (the flux factor)if (!local) waiting += host.flux(waiting);@@ -232,7 +234,7 @@ public class Latency {final StringBuilder s = new StringBuilder(50);// find the minimum waiting time based on the network domain (local or global)- int waiting = agent.minimumDelta;+ int waiting = agent.minimumDelta();s.append("minimumDelta = ").append(waiting);// if we have accessed the domain many times, get slower (the flux factor)diff --git a/source/net/yacy/crawler/data/Transactions.java b/source/net/yacy/crawler/data/Transactions.javaindex 499dae7da..016cd06c6 100644--- a/source/net/yacy/crawler/data/Transactions.java+++ b/source/net/yacy/crawler/data/Transactions.java@@ -1,6 +1,6 @@/*** Transactions- * SPDX-FileCopyrightText: 2014 Michael Peter Christen <[email protected])>+ * SPDX-FileCopyrightText: 2014 Michael Peter Christen <[email protected])>* SPDX-License-Identifier: GPL-2.0-or-later* First released 08.12.2014 at https://yacy.net*@@ -56,7 +56,7 @@ import net.yacy.search.schema.CollectionSchema;**/public class Transactions {-+private final static String XML_PREFIX = "<response>\n<!--\n";private final static char[] WHITESPACE = new char[132];//private final static int WHITESPACE_START = XML_PREFIX.length();@@ -65,14 +65,14 @@ public class Transactions {private static Snapshots inventory = null, archive = null;private static ExecutorService executor = Executors.newCachedThreadPool();private static AtomicInteger executorRunning = new AtomicInteger(0);-+/** the maximum to wait for each wkhtmltopdf call when rendering PDF snapshots */private static long wkhtmltopdfTimeout = 30;-+static {for (int i = 0; i < WHITESPACE.length; i++) WHITESPACE[i] = 32;}-+public static enum State {INVENTORY("inventory"), ARCHIVE("archive"), ANY(null);public String dirname;@@ -80,7 +80,7 @@ public class Transactions {this.dirname = dirname;}}-+/*** @param dir the parent directory of inventory and archive snapshots.* @param wkhtmltopdfSecondsTimeout the maximum to wait for each wkhtmltopdf call when rendering PDF snapshots@@ -94,7 +94,7 @@ public class Transactions {archive = new Snapshots(archiveDir);wkhtmltopdfTimeout = wkhtmltopdfSecondsTimeout;}-+public static synchronized void migrateIPV6Snapshots() {executor.shutdown();try {@@ -122,7 +122,7 @@ public class Transactions {default : Revisions a = inventory.getRevisions(urlhash); return a == null ? archive.getRevisions(urlhash) : a;}}-+/*** get a list of <host>.<port> names in the snapshot directory* @return the list of the given state. if the state is ALL or unknown, all lists are combined@@ -134,12 +134,12 @@ public class Transactions {default : Set<String> a = inventory.listHosts(); a.addAll(archive.listHosts()); return a;}}-+/*** list the snapshots for a given host name* @param hostport the <host>.<port> identifier for the domain (with the same format as applied by the Snapshots.pathToHostPortDir() function).* @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY* @return a map with a set for each depth in the domain of the host name*/public static TreeMap<Integer, Collection<Revisions>> listIDs(final String hostport, final int depth, final State state) {@@ -149,12 +149,12 @@ public class Transactions {default : TreeMap<Integer, Collection<Revisions>> a = inventory.listIDs(hostport, depth); a.putAll(archive.listIDs(hostport, depth)); return a;}}-+/*** get the number of snapshots for the given host name* @param hostport the <host>.<port> identifier for the domain* @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY* @return a count, the total number of documents for the domain and depth*/public static int listIDsSize(final String hostport, final int depth, final State state) {@@ -164,7 +164,7 @@ public class Transactions {default : return inventory.listIDsSize(hostport, depth) + archive.listIDsSize(hostport, depth);}}-+public static boolean store(final SolrInputDocument doc, final boolean concurrency, final boolean loadImage, final boolean replaceOld, final String proxy, final String acceptLanguage) {// GET METADATA FROM DOC@@ -180,7 +180,7 @@ public class Transactions {ConcurrentLog.logException(e);return false;}-+boolean success = loadImage ? store(url, date, depth, concurrency, replaceOld, proxy, acceptLanguage) : true;if (success) {// STORE METADATA FOR THE IMAGE@@ -203,15 +203,15 @@ public class Transactions {success = false;}if(success) {- Transactions.announceStorage(url, depth, date, State.INVENTORY);+ Transactions.announceStorage(url, depth, date, State.INVENTORY);}}-+}-+return success;}-+public static boolean store(final DigestURL url, final Date date, final int depth, final boolean concurrency, final boolean replaceOld, final String proxy, final String acceptLanguage) {@@ -222,12 +222,12 @@ public class Transactions {oldPath.delete();}}-+// STORE METADATA FOR THE IMAGEFile metadataPath = Transactions.definePath(url, depth, date, "xml", Transactions.State.INVENTORY);metadataPath.getParentFile().mkdirs();boolean success = true;-+// STORE AN IMAGEfinal String urls = url.toNormalform(true);final File pdfPath = Transactions.definePath(url, depth, date, "pdf", Transactions.State.INVENTORY);@@ -237,7 +237,7 @@ public class Transactions {public void run() {executorRunning.incrementAndGet();try {- Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent, acceptLanguage, pdfPath, wkhtmltopdfTimeout);+ Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent(), acceptLanguage, pdfPath, wkhtmltopdfTimeout);} catch (Throwable e) {} finally {executorRunning.decrementAndGet();}@@ -245,9 +245,9 @@ public class Transactions {};executor.execute(t);} else {- success = Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent, acceptLanguage, pdfPath, wkhtmltopdfTimeout);+ success = Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent(), acceptLanguage, pdfPath, wkhtmltopdfTimeout);}-+return success;}@@ -270,7 +270,7 @@ public class Transactions {public static Revisions rollback(String urlhash) {return transact(urlhash, State.ARCHIVE, State.INVENTORY);}-+private static Revisions transact(final String urlhash, final State from, final State to) {Revisions r = Transactions.getRevisions(from, urlhash);if (r == null) return null;@@ -303,10 +303,10 @@ public class Transactions {} catch (MalformedURLException e) {ConcurrentLog.logException(e);}-+return null;}-+/*** select a set of urlhashes from the snapshot directory. The selection either ordered* by generation date (upwards == OLDESTFIRST or downwards == LATESTFIRST) or with any@@ -315,7 +315,7 @@ public class Transactions {* @param depth selected depth or null for all depths* @param order Order.ANY, Order.OLDESTFIRST or Order.LATESTFIRST* @param maxcount the maximum number of hosthashes. If unlimited, submit Integer.MAX_VALUE- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY* @return a map of hosthashes with the associated creation date*/public static LinkedHashMap<String, Revisions> select(String host, Integer depth, final Order order, int maxcount, State state) {@@ -332,7 +332,7 @@ public class Transactions {* @param depth* @param date* @param ext- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY* @return a file to the snapshot*/public static File definePath(final DigestURL url, final int depth, final Date date, final String ext, State state) {@@ -345,25 +345,25 @@ public class Transactions {/*** Write information about the storage of a snapshot to the Snapshot-internal index.* The actual writing of files to the target directory must be done elsewehre, this method does not store the snapshot files.- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY* @param url* @param depth* @param date*/public static void announceStorage(final DigestURL url, final int depth, final Date date, State state) {- if (state == State.INVENTORY || state == State.ANY) inventory.announceStorage(url, depth, date);+ if (state == State.INVENTORY || state == State.ANY) inventory.announceStorage(url, depth, date);if (state == State.ARCHIVE || state == State.ANY) archive.announceStorage(url, depth, date);}/*** Delete information about the storage of a snapshot to the Snapshot-internal index.* The actual deletion of files in the target directory must be done elsewehre, this method does not store the snapshot files.- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY* @param url* @param depth*/public static void announceDeletion(final DigestURL url, final int depth, final State state) {- if (state == State.INVENTORY || state == State.ANY) inventory.announceDeletion(url, depth);+ if (state == State.INVENTORY || state == State.ANY) inventory.announceDeletion(url, depth);if (state == State.ARCHIVE || state == State.ANY) archive.announceDeletion(url, depth);}@@ -374,7 +374,7 @@ public class Transactions {* findPaths/3 with a given depth.* @param url* @param ext required extension or null if the extension must not be checked- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY* @return a set of files for snapshots of the url*/public static Collection<File> findPaths(final DigestURL url, final String ext, State state) {@@ -383,14 +383,14 @@ public class Transactions {if (state == State.ARCHIVE || state == State.ANY) result.addAll(archive.findPaths(url, ext));return result;}-+/*** for a given url, get all paths for storage locations.* The locations are all for the single url but may represent different storage times.* @param url* @param ext required extension or null if the extension must not be checked* @param depth- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY* @return a set of files for snapshots of the url*/public static Collection<File> findPaths(final DigestURL url, final int depth, final String ext, State state) {@@ -399,5 +399,5 @@ public class Transactions {if (state == State.ARCHIVE || state == State.ANY) result.addAll(archive.findPaths(url, depth, ext));return result;}-+}diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.javaindex c5e7ef09c..af963473c 100644--- a/source/net/yacy/crawler/retrieval/HTTPLoader.java+++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java@@ -308,7 +308,7 @@ public final class HTTPLoader {private RequestHeader createRequestheader(final Request request, final ClientIdentification.Agent agent)throws IOException {final RequestHeader requestHeader = new RequestHeader();- requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent);+ requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent());if (request.referrerhash() != null) {String refererURL = this.sb.getURL(request.referrerhash());if (refererURL != null) {@@ -478,7 +478,7 @@ public final class HTTPLoader {// create a request headerfinal RequestHeader requestHeader = new RequestHeader();- requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent);+ requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent());requestHeader.put(HeaderFramework.ACCEPT_LANGUAGE, DEFAULT_LANGUAGE);requestHeader.put(HeaderFramework.ACCEPT_ENCODING, DEFAULT_ENCODING);diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.javaindex 2035e961c..0bc5a3928 100644--- a/source/net/yacy/crawler/robots/RobotsTxt.java+++ b/source/net/yacy/crawler/robots/RobotsTxt.java@@ -87,7 +87,7 @@ public class RobotsTxt {public RobotsTxt(final WorkTables worktables, LoaderDispatcher loader, final int maxActiveTheads) {this.threadPool = new ThreadPoolExecutor(maxActiveTheads, maxActiveTheads,0L, TimeUnit.MILLISECONDS,- new LinkedBlockingQueue<Runnable>(),+ new LinkedBlockingQueue<>(),new NamePrefixThreadFactory(RobotsTxt.class.getSimpleName()));this.syncObjects = new ConcurrentHashMap<>();this.tables = worktables;@@ -199,7 +199,7 @@ public class RobotsTxt {if (response == null) {this.processOldEntry(robotsTxt4Host, robotsURL, robotsTable);} else {- robotsTxt4Host = this.processNewEntry(robotsURL, response, agent.robotIDs);+ robotsTxt4Host = this.processNewEntry(robotsURL, response, agent.robotIDs());}}}@@ -225,6 +225,7 @@ public class RobotsTxt {}public void ensureExist(final MultiProtocolURL theURL, final ClientIdentification.Agent agent, boolean concurrent) {+ if (!agent.isRobot()) return;if (theURL.isLocal()) return;final String urlHostPort = getHostPort(theURL);if (urlHostPort == null) return;@@ -266,7 +267,7 @@ public class RobotsTxt {if (response == null) {RobotsTxt.this.processOldEntry(null, robotsURL, robotsTable);} else {- RobotsTxt.this.processNewEntry(robotsURL, response, agent.robotIDs);+ RobotsTxt.this.processNewEntry(robotsURL, response, agent.robotIDs());}}}@@ -292,8 +293,8 @@ public class RobotsTxt {// generate artificial entryrobotsTxt4Host = new RobotsTxtEntry(robotsURL,- new ArrayList<String>(),- new ArrayList<String>(),+ new ArrayList<>(),+ new ArrayList<>(),new Date(),new Date(),null,diff --git a/source/net/yacy/htroot/CrawlCheck_p.java b/source/net/yacy/htroot/CrawlCheck_p.javaindex d32c6f442..c483d36f3 100644--- a/source/net/yacy/htroot/CrawlCheck_p.java+++ b/source/net/yacy/htroot/CrawlCheck_p.java@@ -47,7 +47,7 @@ public class CrawlCheck_p {if (post.containsKey("crawlcheck")) {// get the list of rootURls for this crawl start- final Set<DigestURL> rootURLs = new LinkedHashSet<DigestURL>();+ final Set<DigestURL> rootURLs = new LinkedHashSet<>();final String crawlingStart0 = post.get("crawlingURLs","").trim();final String[] rootURLs0 = crawlingStart0.indexOf('\n') > 0 || crawlingStart0.indexOf('\r') > 0 ? crawlingStart0.split("[\\r\\n]+") : crawlingStart0.split(Pattern.quote("|"));for (String crawlingStart: rootURLs0) {@@ -89,12 +89,12 @@ public class CrawlCheck_p {boolean robotsAllowed = true;if (entry.robotsTxtEntry == null) {prop.put("table_list_" + row + "_robots", "no robots");- prop.put("table_list_" + row + "_crawldelay", agent.minimumDelta + " ms");+ prop.put("table_list_" + row + "_crawldelay", agent.minimumDelta() + " ms");prop.put("table_list_" + row + "_sitemap", "");} else {robotsAllowed = !entry.robotsTxtEntry.isDisallowed(entry.digestURL);prop.put("table_list_" + row + "_robots", "robots exist: " + (robotsAllowed ? "crawl allowed" : "url disallowed"));- prop.put("table_list_" + row + "_crawldelay", Math.max(agent.minimumDelta, entry.robotsTxtEntry.getCrawlDelayMillis()) + " ms");+ prop.put("table_list_" + row + "_crawldelay", Math.max(agent.minimumDelta(), entry.robotsTxtEntry.getCrawlDelayMillis()) + " ms");prop.put("table_list_" + row + "_sitemap", entry.robotsTxtEntry.getSitemaps().toString());}diff --git a/source/net/yacy/htroot/CrawlStartExpert.java b/source/net/yacy/htroot/CrawlStartExpert.javaindex b382d3300..c0d0f3a38 100644--- a/source/net/yacy/htroot/CrawlStartExpert.java+++ b/source/net/yacy/htroot/CrawlStartExpert.java@@ -572,17 +572,15 @@ public class CrawlStartExpert {// ---------- Agent namefinal List<String> agentNames = new ArrayList<>();+ agentNames.add(ClientIdentification.browserAgentName); // see https://github.com/yacy/yacy_search_server/issues/727+ agentNames.add(ClientIdentification.googleAgentName); // see https://github.com/yacy/yacy_search_server/issues/727if (sb.isIntranetMode()) {agentNames.add(ClientIdentification.yacyIntranetCrawlerAgentName);}if (sb.isGlobalMode()) {agentNames.add(ClientIdentification.yacyInternetCrawlerAgentName);}- agentNames.add(ClientIdentification.googleAgentName);- if (sb.isAllIPMode()) {- agentNames.add(ClientIdentification.browserAgentName);- if (ClientIdentification.getAgent(ClientIdentification.customAgentName) != null) agentNames.add(ClientIdentification.customAgentName);- }+ if (ClientIdentification.getAgent(ClientIdentification.customAgentName) != null) agentNames.add(ClientIdentification.customAgentName);String defaultAgentName = agentNames.get(0);if (post != null && post.containsKey("agentName")) {final String agentName = post.get("agentName", sb.isIntranetMode() ? ClientIdentification.yacyIntranetCrawlerAgentName : ClientIdentification.yacyInternetCrawlerAgentName);diff --git a/source/net/yacy/htroot/Network.java b/source/net/yacy/htroot/Network.javaindex 64519f686..b8cc05390 100644--- a/source/net/yacy/htroot/Network.java+++ b/source/net/yacy/htroot/Network.java@@ -222,7 +222,7 @@ public class Network {return prop;}- final ConcurrentMap<String, String> map = new ConcurrentHashMap<String, String>();+ final ConcurrentMap<String, String> map = new ConcurrentHashMap<>();final String challengeIP = post.get("peerIP");final String challengePort = post.get("peerPort");map.put(Seed.IP, challengeIP);@@ -302,10 +302,10 @@ public class Network {}// find updated Information using YaCyNews- final HashSet<String> updatedProfile = new HashSet<String>();- final HashMap<String, Map<String, String>> updatedWiki = new HashMap<String, Map<String, String>>();- final HashMap<String, Map<String, String>> updatedBlog = new HashMap<String, Map<String, String>>();- final HashMap<String, String> isCrawling = new HashMap<String, String>();+ final HashSet<String> updatedProfile = new HashSet<>();+ final HashMap<String, Map<String, String>> updatedWiki = new HashMap<>();+ final HashMap<String, Map<String, String>> updatedBlog = new HashMap<>();+ final HashMap<String, String> isCrawling = new HashMap<>();NewsDB.Record record;final Iterator<NewsDB.Record> recordIterator = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);while (recordIterator.hasNext()) {@@ -430,7 +430,7 @@ public class Network {prop.put(STR_TABLE_LIST + conCount + "_ssl_portssl", seed.get(Seed.PORTSSL,"8443"));userAgent = null;if (seed.hash != null && seed.hash.equals(sb.peers.mySeed().hash)) {- userAgent = ClientIdentification.yacyInternetCrawlerAgent.userAgent;+ userAgent = ClientIdentification.yacyInternetCrawlerAgent.userAgent();location = ClientIdentification.generateLocation();} else {userAgent = sb.peers.peerActions.getUserAgent(ip);diff --git a/source/net/yacy/htroot/api/latency_p.java b/source/net/yacy/htroot/api/latency_p.javaindex c799a32d5..aedf5e048 100644--- a/source/net/yacy/htroot/api/latency_p.java+++ b/source/net/yacy/htroot/api/latency_p.java@@ -55,7 +55,7 @@ public class latency_p {prop.put("domains_" + c + "_count", host.count());prop.put("domains_" + c + "_average", host.average());prop.put("domains_" + c + "_robots", host.robotsDelay());- prop.put("domains_" + c + "_flux", host.flux(agent.minimumDelta));+ prop.put("domains_" + c + "_flux", host.flux(agent.minimumDelta()));c++;}prop.put("domains", c);diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.javaindex ca0cd0a35..38c82d210 100644--- a/source/net/yacy/peers/SeedDB.java+++ b/source/net/yacy/peers/SeedDB.java@@ -122,9 +122,9 @@ public final class SeedDB implements AlternativeDomainNames {this.seedPotentialDB = openSeedTable(this.seedPotentialDBFile);// check if we are in the seedCaches: this can happen if someone else published our seed- removeMySeed();+ this.removeMySeed();- this.lastSeedUpload_seedDBSize = sizeConnected();+ this.lastSeedUpload_seedDBSize = this.sizeConnected();// create or init news databasethis.newsPool = new NewsPool(networkRoot, useTailCache, exceed134217727);@@ -154,7 +154,7 @@ public final class SeedDB implements AlternativeDomainNames {// replace my (old) seed with new seed definition from other network// but keep the seed name- final String peername = myName();+ final String peername = this.myName();this.mySeed = null; // my own seedthis.myOwnSeedFile = new File(newNetworkRoot, SeedDB.DBFILE_OWN_SEED);@@ -166,13 +166,13 @@ public final class SeedDB implements AlternativeDomainNames {this.seedPassiveDB = openSeedTable(this.seedPassiveDBFile);this.seedPotentialDB = openSeedTable(this.seedPotentialDBFile);- initMySeed();+ this.initMySeed();this.mySeed.setName(peername);// check if we are in the seedCaches: this can happen if someone else published our seed- removeMySeed();+ this.removeMySeed();- this.lastSeedUpload_seedDBSize = sizeConnected();+ this.lastSeedUpload_seedDBSize = this.sizeConnected();// create or init news databasethis.newsPool = new NewsPool(newNetworkRoot, useTailCache, exceed134217727);@@ -227,49 +227,49 @@ public final class SeedDB implements AlternativeDomainNames {public Seed mySeed() {if (this.mySeed == null) {- if (sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init- initMySeed();+ if (this.sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init+ this.initMySeed();}return this.mySeed;}public void setMyName(final String name) {- mySeed().setName(name);+ this.mySeed().setName(name);}@Overridepublic String myAlternativeAddress() {- return mySeed().getName() + ".yacy";+ return this.mySeed().getName() + ".yacy";}@Deprecatedpublic String myIP() {- return mySeed().getIP();+ return this.mySeed().getIP();}-+@Overridepublic Set<String> myIPs() {- return mySeed().getIPs();+ return this.mySeed().getIPs();}@Overridepublic int myPort() {- return mySeed().getPort();+ return this.mySeed().getPort();}@Overridepublic String myName() {- return mySeed().getName();+ return this.mySeed().getName();}@Overridepublic String myID() {- return mySeed().hash;+ return this.mySeed().hash;}public synchronized void removeMySeed() {if (this.seedActiveDB.isEmpty() && this.seedPassiveDB.isEmpty() && this.seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early- if (this.mySeed == null) initMySeed();+ if (this.mySeed == null) this.initMySeed();try {final byte[] mySeedHash = ASCII.getBytes(this.mySeed.hash);this.seedActiveDB.delete(mySeedHash);@@ -280,13 +280,13 @@ public final class SeedDB implements AlternativeDomainNames {public void saveMySeed() {try {- mySeed().save(this.myOwnSeedFile);+ this.mySeed().save(this.myOwnSeedFile);} catch (final IOException e) { ConcurrentLog.warn("yacySeedDB", "could not save mySeed '"+ this.myOwnSeedFile +"': "+ e.getMessage()); }}public boolean noDHTActivity() {// for small networks, we don't perform DHT transmissions, because it is possible to search over all peers- return sizeConnected() <= dhtActivityMagic;+ return this.sizeConnected() <= dhtActivityMagic;}private synchronized static MapDataMining openSeedTable(final File seedDBFile) {@@ -323,9 +323,9 @@ public final class SeedDB implements AlternativeDomainNames {return seedDB;}- public synchronized void resetActiveTable() { this.seedActiveDB = resetSeedTable(this.seedActiveDB, this.seedActiveDBFile); }- private synchronized void resetPassiveTable() { this.seedPassiveDB = resetSeedTable(this.seedPassiveDB, this.seedPassiveDBFile); }- private synchronized void resetPotentialTable() { this.seedPotentialDB = resetSeedTable(this.seedPotentialDB, this.seedPotentialDBFile); }+ public synchronized void resetActiveTable() { this.seedActiveDB = this.resetSeedTable(this.seedActiveDB, this.seedActiveDBFile); }+ private synchronized void resetPassiveTable() { this.seedPassiveDB = this.resetSeedTable(this.seedPassiveDB, this.seedPassiveDBFile); }+ private synchronized void resetPotentialTable() { this.seedPotentialDB = this.resetSeedTable(this.seedPotentialDB, this.seedPotentialDBFile); }public void close() {if (this.seedActiveDB != null) this.seedActiveDB.close();@@ -374,7 +374,7 @@ public final class SeedDB implements AlternativeDomainNames {if (yacydom.endsWith(".yacyh")) {// find a peer with its hexhashhash = Seed.hexHash2b64Hash(yacydom.substring(0, yacydom.length() - 6));- seed = get(hash);+ seed = this.get(hash);if (seed == null) {Network.log.warn("cluster peer '" + yacydom + "' was not found.");} else {@@ -382,7 +382,7 @@ public final class SeedDB implements AlternativeDomainNames {}} else if (yacydom.endsWith(".yacy")) {// find a peer with its name- seed = lookupByName(yacydom.substring(0, yacydom.length() - 5));+ seed = this.lookupByName(yacydom.substring(0, yacydom.length() - 5));if (seed == null) {Network.log.warn("cluster peer '" + yacydom + "' was not found.");} else {@@ -407,14 +407,14 @@ public final class SeedDB implements AlternativeDomainNames {public Seed anySeedVersion(final double minVersion) {// return just any seed that has a specific minimum version number- final Iterator<Seed> e = seedsConnected(true, true, Seed.randomHash(), minVersion);+ final Iterator<Seed> e = this.seedsConnected(true, true, Seed.randomHash(), minVersion);return e.next();}private final Map<Long, Integer> sizeActiveSince_result = new ConcurrentHashMap<>();private final Map<Long, Long> sizeActiveSince_time = new ConcurrentHashMap<>();-+/*** count the number of peers that had been seen within the given time limit* @param limitMinutes the time limit in minutes. 1440 minutes is a day@@ -422,22 +422,22 @@ public final class SeedDB implements AlternativeDomainNames {*/public int sizeActiveSince(final long limitMinutes) {// buffering because that iteration takes a long time- long now = System.currentTimeMillis();- Integer s0 = sizeActiveSince_result.get(limitMinutes);- Long t0 = sizeActiveSince_time.get(limitMinutes);+ final long now = System.currentTimeMillis();+ final Integer s0 = this.sizeActiveSince_result.get(limitMinutes);+ final Long t0 = this.sizeActiveSince_time.get(limitMinutes);if (s0 == null || t0 == null || now - t0 > 60000) {- int s1 = sizeActiveSinceInt(limitMinutes);- sizeActiveSince_result.put(limitMinutes, s1);- sizeActiveSince_time.put(limitMinutes, now);+ final int s1 = this.sizeActiveSinceInt(limitMinutes);+ this.sizeActiveSince_result.put(limitMinutes, s1);+ this.sizeActiveSince_time.put(limitMinutes, now);return s1;}- return s0;+ return s0;}-+private int sizeActiveSinceInt(final long limitMinutes) {int c = this.seedActiveDB.size();Seed seed;- Iterator<Seed> i = seedsSortedDisconnected(false, Seed.LASTSEEN);+ Iterator<Seed> i = this.seedsSortedDisconnected(false, Seed.LASTSEEN);while (i.hasNext()) {seed = i.next();if (seed != null) {@@ -445,7 +445,7 @@ public final class SeedDB implements AlternativeDomainNames {c++;}}- i = seedsSortedPotential(false, Seed.LASTSEEN);+ i = this.seedsSortedPotential(false, Seed.LASTSEEN);while (i.hasNext()) {seed = i.next();if (seed != null) {@@ -476,7 +476,7 @@ public final class SeedDB implements AlternativeDomainNames {public long countPassiveRWI() { return this.seedPassiveDB.getLongAcc(Seed.ICOUNT); }public long countPotentialURL() { return this.seedPotentialDB.getLongAcc(Seed.LCOUNT); }public long countPotentialRWI() { return this.seedPotentialDB.getLongAcc(Seed.ICOUNT); }-+public void addConnected(final Seed seed) {if (seed.isProper(false) != null) return;//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));@@ -488,7 +488,7 @@ public final class SeedDB implements AlternativeDomainNames {this.seedPotentialDB.delete(ASCII.getBytes(seed.hash));} catch (final Exception e) {Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- resetActiveTable();+ this.resetActiveTable();}}}@@ -506,7 +506,7 @@ public final class SeedDB implements AlternativeDomainNames {this.seedPassiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);} catch (final Exception e) {Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- resetPassiveTable();+ this.resetPassiveTable();}}}@@ -524,7 +524,7 @@ public final class SeedDB implements AlternativeDomainNames {this.seedPotentialDB.insert(ASCII.getBytes(seed.hash), seedPropMap);} catch (final Exception e) {Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- resetPotentialTable();+ this.resetPotentialTable();}}}@@ -558,7 +558,7 @@ public final class SeedDB implements AlternativeDomainNames {private Seed get(final String hash, final MapDataMining database) {if (hash == null || hash.isEmpty()) return null;if ((this.mySeed != null) && (hash.equals(this.mySeed.hash))) return this.mySeed;- final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();+ final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<>();try {final Map<String, String> map = database.get(ASCII.getBytes(hash));if (map == null) return null;@@ -584,7 +584,7 @@ public final class SeedDB implements AlternativeDomainNames {private Seed get(final byte[] hash, final MapDataMining database) {if (hash == null || hash.length == 0) return null;if ((this.mySeed != null) && (ASCII.String(hash).equals(this.mySeed.hash))) return this.mySeed;- final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();+ final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<>();try {final Map<String, String> map = database.get(hash);if (map == null) return null;@@ -600,33 +600,33 @@ public final class SeedDB implements AlternativeDomainNames {}public Seed getConnected(final String hash) {- return get(hash, this.seedActiveDB);+ return this.get(hash, this.seedActiveDB);}public Seed getConnected(final byte[] hash) {- return get(hash, this.seedActiveDB);+ return this.get(hash, this.seedActiveDB);}public Seed getDisconnected(final String hash) {- return get(hash, this.seedPassiveDB);+ return this.get(hash, this.seedPassiveDB);}public Seed getDisconnected(final byte[] hash) {- return get(hash, this.seedPassiveDB);+ return this.get(hash, this.seedPassiveDB);}public Seed getPotential(final String hash) {- return get(hash, this.seedPotentialDB);+ return this.get(hash, this.seedPotentialDB);}public Seed getPotential(final byte[] hash) {- return get(hash, this.seedPotentialDB);+ return this.get(hash, this.seedPotentialDB);}public Seed get(final String hash) {- Seed seed = getConnected(hash);- if (seed == null) seed = getDisconnected(hash);- if (seed == null) seed = getPotential(hash);+ Seed seed = this.getConnected(hash);+ if (seed == null) seed = this.getDisconnected(hash);+ if (seed == null) seed = this.getPotential(hash);return seed;}@@ -638,7 +638,7 @@ public final class SeedDB implements AlternativeDomainNames {this.seedActiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);} catch (final Exception e) {Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- resetActiveTable();+ this.resetActiveTable();}}}@@ -650,7 +650,7 @@ public final class SeedDB implements AlternativeDomainNames {this.seedPassiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);} catch (final Exception e) {Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- resetActiveTable();+ this.resetActiveTable();}}}@@ -662,7 +662,7 @@ public final class SeedDB implements AlternativeDomainNames {this.seedPotentialDB.insert(ASCII.getBytes(seed.hash), seedPropMap);} catch (final Exception e) {Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- resetActiveTable();+ this.resetActiveTable();}}}@@ -673,7 +673,7 @@ public final class SeedDB implements AlternativeDomainNames {// local peer?if (peerName.equals("localpeer")) {- if (this.mySeed == null) initMySeed();+ if (this.mySeed == null) this.initMySeed();return this.mySeed;}@@ -683,8 +683,8 @@ public final class SeedDB implements AlternativeDomainNames {// enumerate the cacheString name = Seed.checkPeerName(peerName);synchronized (this) { try {- Collection<byte[]> idx = this.seedActiveDB.select(Seed.NAME, name);- for (byte[] pk: idx) {+ final Collection<byte[]> idx = this.seedActiveDB.select(Seed.NAME, name);+ for (final byte[] pk: idx) {seed = this.getConnected(pk);if (seed == null) continue;//System.out.println("*** found lookupByName in seedActiveDB: " + peerName);@@ -693,8 +693,8 @@ public final class SeedDB implements AlternativeDomainNames {} catch (final IOException e ) {}}synchronized (this) { try {- Collection<byte[]> idx = this.seedPassiveDB.select(Seed.NAME, name);- for (byte[] pk: idx) {+ final Collection<byte[]> idx = this.seedPassiveDB.select(Seed.NAME, name);+ for (final byte[] pk: idx) {seed = this.getDisconnected(pk);if (seed == null) continue;//System.out.println("*** found lookupByName in seedPassiveDB: " + peerName);@@ -704,7 +704,7 @@ public final class SeedDB implements AlternativeDomainNames {}}// check local seed- if (this.mySeed == null) initMySeed();+ if (this.mySeed == null) this.initMySeed();name = this.mySeed.getName().toLowerCase();if (name.equals(peerName)) return this.mySeed;// nothing found@@ -718,13 +718,13 @@ public final class SeedDB implements AlternativeDomainNames {final boolean lookupDisconnected,final boolean lookupPotential) {- for (String i: peerIPs) {- Seed s = lookupByIP(Domains.dnsResolve(i), port, lookupConnected, lookupDisconnected, lookupPotential);+ for (final String i: peerIPs) {+ final Seed s = this.lookupByIP(Domains.dnsResolve(i), port, lookupConnected, lookupDisconnected, lookupPotential);if (s != null) return s;}return null;}-+public Seed lookupByIP(final InetAddress peerIP,final int port, /* port may be -1 if not significant */@@ -737,18 +737,18 @@ public final class SeedDB implements AlternativeDomainNames {// local peer?if (Domains.isThisHostIP(peerIP)) {- if (this.mySeed == null) initMySeed();+ if (this.mySeed == null) this.initMySeed();if (port > 0 && this.mySeed.getPort() == port) return this.mySeed;}// then try to use the cacheSeed seed = null;- String ipString = peerIP.getHostAddress();+ final String ipString = peerIP.getHostAddress();if (lookupConnected) {try {- Collection<byte[]> idx = this.seedActiveDB.select(Seed.IP, ipString);- for (byte[] pk: idx) {+ final Collection<byte[]> idx = this.seedActiveDB.select(Seed.IP, ipString);+ for (final byte[] pk: idx) {seed = this.getConnected(pk);if (seed == null) continue;if (port > 0 && seed.getPort() != port) continue;@@ -762,8 +762,8 @@ public final class SeedDB implements AlternativeDomainNames {if (lookupDisconnected) {try {- Collection<byte[]> idx = this.seedPassiveDB.select(Seed.IP, ipString);- for (byte[] pk: idx) {+ final Collection<byte[]> idx = this.seedPassiveDB.select(Seed.IP, ipString);+ for (final byte[] pk: idx) {seed = this.getDisconnected(pk);if (seed == null) continue;if (port > 0 && seed.getPort() != port) continue;@@ -777,8 +777,8 @@ public final class SeedDB implements AlternativeDomainNames {if (lookupPotential) {try {- Collection<byte[]> idx = this.seedPotentialDB.select(Seed.IP, ipString);- for (byte[] pk: idx) {+ final Collection<byte[]> idx = this.seedPotentialDB.select(Seed.IP, ipString);+ for (final byte[] pk: idx) {seed = this.getPotential(pk);if (seed == null) continue;if (port > 0 && seed.getPort() != port) continue;@@ -792,7 +792,7 @@ public final class SeedDB implements AlternativeDomainNames {// check local seedif (this.mySeed == null || !this.mySeed.getIPs().contains(ipString)) return null;- int p = this.mySeed.getPort();+ final int p = this.mySeed.getPort();if (port > 0 && p != port) return null;//System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());return this.mySeed;@@ -800,13 +800,13 @@ public final class SeedDB implements AlternativeDomainNames {private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException {PrintWriter pw = null;- final ArrayList<String> v = new ArrayList<String>(this.seedActiveDB.size() + 1);+ final ArrayList<String> v = new ArrayList<>(this.seedActiveDB.size() + 1);try {pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));- List<Seed> seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed, false, 0.0f);+ final List<Seed> seedlist = this.getSeedlist(Integer.MAX_VALUE, addMySeed, false, 0.0f);String line;- for (Seed seed: seedlist) {+ for (final Seed seed: seedlist) {line = seed.genSeedStr(null);v.add(line);pw.print(line + serverCore.CRLF_STRING);@@ -819,8 +819,8 @@ public final class SeedDB implements AlternativeDomainNames {}public ArrayList<Seed> getSeedlist(int maxcount, boolean addMySeed, boolean nodeonly, float minversion) {- final ArrayList<Seed> v = new ArrayList<Seed>(this.seedActiveDB.size() + 1000);-+ final ArrayList<Seed> v = new ArrayList<>(this.seedActiveDB.size() + 1000);+// store own peer seedif (addMySeed) v.add(this.mySeed);@@ -841,12 +841,12 @@ public final class SeedDB implements AlternativeDomainNames {}final StringBuilder encoded = new StringBuilder(1024);- for (Seed seed: v) {+ for (final Seed seed: v) {encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);}return v;}-+protected String uploadSeedList(final yacySeedUploader uploader,final serverSwitch sb,final SeedDB seedDB,@@ -861,7 +861,7 @@ public final class SeedDB implements AlternativeDomainNames {// create a seed file which for uploading ...seedFile = File.createTempFile("seedFile",".txt", seedDB.myOwnSeedFile.getParentFile());if (Network.log.isFine()) Network.log.fine("SaveSeedList: Storing seedlist into tempfile " + seedFile.toString());- final ArrayList<String> uv = storeSeedList(seedFile, true);+ final ArrayList<String> uv = this.storeSeedList(seedFile, true);// uploading the seed fileif (Network.log.isFine()) Network.log.fine("SaveSeedList: Trying to upload seed-file, " + seedFile.length() + " bytes, " + uv.size() + " entries.");@@ -895,7 +895,7 @@ public final class SeedDB implements AlternativeDomainNames {final RequestHeader reqHeader = new RequestHeader();reqHeader.put(HeaderFramework.PRAGMA, "no-cache");reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache, no-store"); // httpc uses HTTP/1.0 is this necessary?- reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);+ reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent());byte[] content = null;try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {@@ -906,7 +906,7 @@ public final class SeedDB implements AlternativeDomainNames {} catch (final Exception e) {throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());}-+// check response codeif (client.getHttpResponse().getStatusLine().getStatusCode() != 200) {throw new IOException("Server returned status: " + client.getHttpResponse().getStatusLine());@@ -967,10 +967,10 @@ public final class SeedDB implements AlternativeDomainNames {hash = Seed.hexHash2b64Hash(hash);}// check remote seeds- seed = getConnected(hash); // checks only remote, not local+ seed = this.getConnected(hash); // checks only remote, not local// check local seedif (seed == null) {- if (this.mySeed == null) initMySeed();+ if (this.mySeed == null) this.initMySeed();if (hash.equals(this.mySeed.hash))seed = this.mySeed;else return null;@@ -989,9 +989,9 @@ public final class SeedDB implements AlternativeDomainNames {}// identify domainfinal String domain = host.substring(0, host.length() - 5).toLowerCase(Locale.ROOT);- seed = lookupByName(domain);+ seed = this.lookupByName(domain);if (seed == null) return null;- if (this.mySeed == null) initMySeed();+ if (this.mySeed == null) this.initMySeed();if (seed == this.mySeed && !(seed.isOnline())) {// take local ip instead of externalfinal Set<String> ips = Switchboard.getSwitchboard().myPublicIPs();@@ -1024,7 +1024,7 @@ public final class SeedDB implements AlternativeDomainNames {this.it = (firstKey == null) ? database.entries(up, rot) : database.entries(up, rot, firstKey, secondKey);double version;while (true) {- this.nextSeed = internalNext();+ this.nextSeed = this.internalNext();if (this.nextSeed == null) break;version = this.nextSeed.getVersion();if (version >= this.minVersion || version == 0.0) break; // include 0.0 to access always developer peers@@ -1032,14 +1032,14 @@ public final class SeedDB implements AlternativeDomainNames {} catch (final IOException e) {ConcurrentLog.logException(e);Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);- if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);+ if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);+ if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);this.it = null;} catch (final kelondroException e) {ConcurrentLog.logException(e);Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);- if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);+ if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);+ if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);this.it = null;}}@@ -1048,13 +1048,13 @@ public final class SeedDB implements AlternativeDomainNames {this.database = database;try {this.it = database.entries(up, field);- this.nextSeed = internalNext();+ this.nextSeed = this.internalNext();} catch (final kelondroException e) {ConcurrentLog.logException(e);Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);- if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);- if (database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);+ if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);+ if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);+ if (database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);this.it = null;}}@@ -1084,9 +1084,9 @@ public final class SeedDB implements AlternativeDomainNames {} catch (final Exception e) {ConcurrentLog.logException(e);Network.log.severe("ERROR internalNext: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);- if (this.database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);- if (this.database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);- if (this.database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);+ if (this.database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);+ if (this.database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);+ if (this.database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);return null;}}@@ -1096,7 +1096,7 @@ public final class SeedDB implements AlternativeDomainNames {final Seed seed = this.nextSeed;double version;try {while (true) {- this.nextSeed = internalNext();+ this.nextSeed = this.internalNext();if (this.nextSeed == null) break;version = this.nextSeed.getVersion();if (version >= this.minVersion || version == 0.0) break; // include 0.0 to access always developer peers@@ -1121,12 +1121,12 @@ public final class SeedDB implements AlternativeDomainNames {public void loadSeedListConcurrently(final String seedListFileURL, final AtomicInteger scc, final int timeout, final boolean checkAge) {// uses the superseed to initialize the database with known seeds- Thread seedLoader = new Thread("SeedDB.loadSeedListConcurrently") {+ final Thread seedLoader = new Thread("SeedDB.loadSeedListConcurrently") {@Overridepublic void run() {// load the seed listtry (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout)) {- DigestURL url = new DigestURL(seedListFileURL);+ final DigestURL url = new DigestURL(seedListFileURL);//final long start = System.currentTimeMillis();final RequestHeader reqHeader = new RequestHeader();reqHeader.put(HeaderFramework.PRAGMA, "no-cache");@@ -1134,8 +1134,8 @@ public final class SeedDB implements AlternativeDomainNames {client.setHeader(reqHeader.entrySet());client.HEADResponse(url.toNormalform(false), false);- int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();- ResponseHeader header = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());+ final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();+ final ResponseHeader header = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());if (checkAge) {if (header.lastModified() == null) {Network.log.warn("BOOTSTRAP: seed-list URL " + seedListFileURL + " not usable, last-modified is missing");@@ -1147,17 +1147,17 @@ public final class SeedDB implements AlternativeDomainNames {}scc.incrementAndGet();final byte[] content = client.GETbytes(url, null, null, false);- Iterator<String> enu = FileUtils.strings(content);+ final Iterator<String> enu = FileUtils.strings(content);int lc = 0;int uptodatec = 0, outdatedc = 0;while (enu.hasNext()) {try {- Seed ys = Seed.genRemoteSeed(enu.next(), false, null);- if ((ys != null) && (!mySeedIsDefined() || !mySeed().hash.equals(ys.hash))) {+ final Seed ys = Seed.genRemoteSeed(enu.next(), false, null);+ if ((ys != null) && (!SeedDB.this.mySeedIsDefined() || !SeedDB.this.mySeed().hash.equals(ys.hash))) {final long lastseen = Math.abs((System.currentTimeMillis() - ys.getLastSeenUTC()) / 1000 / 60);if (lastseen < 240) uptodatec++; else outdatedc++;if ( lastseen < 240 || lc < 10 ) {- if (peerActions.connectPeer(ys, false) ) lc++;+ if (SeedDB.this.peerActions.connectPeer(ys, false) ) lc++;}}} catch (final Throwable e ) {@@ -1177,5 +1177,5 @@ public final class SeedDB implements AlternativeDomainNames {};seedLoader.start();}-+}diff --git a/source/net/yacy/repository/LoaderDispatcher.java b/source/net/yacy/repository/LoaderDispatcher.javaindex 1d1439482..2b3ce1c0d 100644--- a/source/net/yacy/repository/LoaderDispatcher.java+++ b/source/net/yacy/repository/LoaderDispatcher.java@@ -71,7 +71,7 @@ public final class LoaderDispatcher {private final static int accessTimeMaxsize = 1000;private final static ConcurrentLog log = new ConcurrentLog("LOADER");- private static final ConcurrentHashMap<String, Long> accessTime = new ConcurrentHashMap<String, Long>(); // to protect targets from DDoS+ private static final ConcurrentHashMap<String, Long> accessTime = new ConcurrentHashMap<>(); // to protect targets from DDoSprivate final Switchboard sb;private final HashSet<String> supportedProtocols;@@ -83,14 +83,14 @@ public final class LoaderDispatcher {public LoaderDispatcher(final Switchboard sb) {this.sb = sb;- this.supportedProtocols = new HashSet<String>(Arrays.asList(new String[]{"http","https","ftp","smb","file"}));+ this.supportedProtocols = new HashSet<>(Arrays.asList("http", "https", "ftp", "smb", "file"));// initiate loader objectsthis.httpLoader = new HTTPLoader(sb, LoaderDispatcher.log);this.ftpLoader = new FTPLoader(sb, LoaderDispatcher.log);this.smbLoader = new SMBLoader(sb, LoaderDispatcher.log);this.fileLoader = new FileLoader(sb, LoaderDispatcher.log);- this.loaderSteering = new ConcurrentHashMap<DigestURL, Semaphore>();+ this.loaderSteering = new ConcurrentHashMap<>();}public boolean isSupportedProtocol(final String protocol) {@@ -115,7 +115,7 @@ public final class LoaderDispatcher {final boolean forText,final boolean global) {- CrawlProfile profile =+ final CrawlProfile profile =(forText) ?((global) ?this.sb.crawler.defaultTextSnippetGlobalProfile :@@ -137,7 +137,7 @@ public final class LoaderDispatcher {public void load(final DigestURL url, final CacheStrategy cacheStratgy, final int maxFileSize, final File targetFile, BlacklistType blacklistType, ClientIdentification.Agent agent) throws IOException {- final byte[] b = load(request(url, false, true), cacheStratgy, maxFileSize, blacklistType, agent).getContent();+ final byte[] b = this.load(this.request(url, false, true), cacheStratgy, maxFileSize, blacklistType, agent).getContent();if (b == null) throw new IOException("load == null");final File tmp = new File(targetFile.getAbsolutePath() + ".tmp");@@ -149,7 +149,7 @@ public final class LoaderDispatcher {}public Response load(final Request request, final CacheStrategy cacheStrategy, final BlacklistType blacklistType, ClientIdentification.Agent agent) throws IOException {- return load(request, cacheStrategy, protocolMaxFileSize(request.url()), blacklistType, agent);+ return this.load(request, cacheStrategy, this.protocolMaxFileSize(request.url()), blacklistType, agent);}/**@@ -169,7 +169,7 @@ public final class LoaderDispatcher {if (check != null && cacheStrategy != CacheStrategy.NOCACHE) {// a loading process is going on for that url//ConcurrentLog.info("LoaderDispatcher", "waiting for " + request.url().toNormalform(true));- long t = System.currentTimeMillis();+ final long t = System.currentTimeMillis();try { check.tryAcquire(5, TimeUnit.SECONDS);} catch (final InterruptedException e) {}ConcurrentLog.info("LoaderDispatcher", "waited " + (System.currentTimeMillis() - t) + " ms for " + request.url().toNormalform(true));// now the process may have terminated and we run a normal loading@@ -178,7 +178,7 @@ public final class LoaderDispatcher {this.loaderSteering.put(request.url(), new Semaphore(0));try {- final Response response = loadInternal(request, cacheStrategy, maxFileSize, blacklistType, agent);+ final Response response = this.loadInternal(request, cacheStrategy, maxFileSize, blacklistType, agent);// finally block cleans up loaderSteering and semaphorereturn response;} catch (final IOException e) {@@ -215,7 +215,7 @@ public final class LoaderDispatcher {}// check if we have the page in the cache- Response response = loadFromCache(request, cacheStrategy, agent, url, crawlProfile);+ Response response = this.loadFromCache(request, cacheStrategy, agent, url, crawlProfile);if(response != null) {return response;}@@ -230,7 +230,7 @@ public final class LoaderDispatcher {// check access time: this is a double-check (we checked possibly already in the balancer)// to make sure that we don't DoS the target by mistake- checkAccessTime(agent, url);+ this.checkAccessTime(agent, url);// now it's for sure that we will access the target. Remember the access timeif (host != null) {@@ -305,7 +305,7 @@ public final class LoaderDispatcher {// create request header values and a response object because we need that// in case that we want to return the cached content in the next stepfinal RequestHeader requestHeader = new RequestHeader();- requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent);+ requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent());String refererURL = null;if (request.referrerhash() != null) refererURL = this.sb.getURL(request.referrerhash());if (refererURL != null) requestHeader.put(RequestHeader.REFERER, refererURL);@@ -373,7 +373,7 @@ public final class LoaderDispatcher {}// check if we have the page in the cache- Response cachedResponse = loadFromCache(request, cacheStrategy, agent, url, crawlProfile);+ final Response cachedResponse = this.loadFromCache(request, cacheStrategy, agent, url, crawlProfile);if (cachedResponse != null) {return new StreamResponse(cachedResponse, new ByteArrayInputStream(cachedResponse.getContent()));}@@ -388,7 +388,7 @@ public final class LoaderDispatcher {// check access time: this is a double-check (we checked possibly already in the balancer)// to make sure that we don't DoS the target by mistake- checkAccessTime(agent, url);+ this.checkAccessTime(agent, url);// now it's for sure that we will access the target. Remember the access timeif (host != null) {@@ -422,18 +422,18 @@ public final class LoaderDispatcher {*/private void checkAccessTime(ClientIdentification.Agent agent, final DigestURL url) {if (!url.isLocal()) {- String host = url.getHost();+ final String host = url.getHost();final Long lastAccess = accessTime.get(host);long wait = 0;if (lastAccess != null)- wait = Math.max(0, agent.minimumDelta + lastAccess.longValue() - System.currentTimeMillis());+ wait = Math.max(0, agent.minimumDelta() + lastAccess.longValue() - System.currentTimeMillis());if (wait > 0) {// force a sleep here. Instead just sleep we clean up the// accessTime mapfinal long untilTime = System.currentTimeMillis() + wait;cleanupAccessTimeTable(untilTime);if (System.currentTimeMillis() < untilTime) {- long frcdslp = untilTime - System.currentTimeMillis();+ final long frcdslp = untilTime - System.currentTimeMillis();LoaderDispatcher.log.info("Forcing sleep of " + frcdslp + " ms for host " + host);try {Thread.sleep(frcdslp);@@ -474,7 +474,7 @@ public final class LoaderDispatcher {*/public byte[] loadContent(final Request request, final CacheStrategy cacheStrategy, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException {// try to download the resource using the loader- final Response entry = load(request, cacheStrategy, blacklistType, agent);+ final Response entry = this.load(request, cacheStrategy, blacklistType, agent);if (entry == null) return null; // not found in web// read resource body (if it is there)@@ -498,7 +498,7 @@ public final class LoaderDispatcher {Semaphore check = this.loaderSteering.get(request.url());if (check != null && cacheStrategy != CacheStrategy.NOCACHE) {// a loading process is going on for that url- long t = System.currentTimeMillis();+ final long t = System.currentTimeMillis();try {check.tryAcquire(5, TimeUnit.SECONDS);} catch (final InterruptedException e) {@@ -511,8 +511,8 @@ public final class LoaderDispatcher {this.loaderSteering.put(request.url(), new Semaphore(0));try {- response = openInputStreamInternal(request, cacheStrategy, maxFileSize, blacklistType, agent);- } catch(IOException ioe) {+ response = this.openInputStreamInternal(request, cacheStrategy, maxFileSize, blacklistType, agent);+ } catch(final IOException ioe) {/* Do not re encapsulate any eventual IOException in an IOException */throw ioe;} catch (final Throwable e) {@@ -539,14 +539,14 @@ public final class LoaderDispatcher {*/public StreamResponse openInputStream(final Request request, final CacheStrategy cacheStrategy,BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException {- final int maxFileSize = protocolMaxFileSize(request.url());+ final int maxFileSize = this.protocolMaxFileSize(request.url());return this.openInputStream(request, cacheStrategy, blacklistType, agent, maxFileSize);}public Document[] loadDocuments(final Request request, final CacheStrategy cacheStrategy, final int maxFileSize, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException, Parser.Failure {// load resource- final Response response = load(request, cacheStrategy, maxFileSize, blacklistType, agent);+ final Response response = this.load(request, cacheStrategy, maxFileSize, blacklistType, agent);final DigestURL url = request.url();if (response == null) throw new IOException("no Response for url " + url);@@ -554,11 +554,11 @@ public final class LoaderDispatcher {if (response.getContent() == null || response.getResponseHeader() == null) throw new IOException("no Content available for url " + url);// parse resource- Document[] documents = response.parse();+ final Document[] documents = response.parse();- String x_robots_tag = response.getResponseHeader().getXRobotsTag();+ final String x_robots_tag = response.getResponseHeader().getXRobotsTag();if (x_robots_tag.indexOf("noindex",0) >= 0) {- for (Document d: documents) d.setIndexingDenied(true);+ for (final Document d: documents) d.setIndexingDenied(true);}return documents;@@ -566,7 +566,7 @@ public final class LoaderDispatcher {public Document loadDocument(final DigestURL location, final CacheStrategy cachePolicy, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException {// load resource- Request request = request(location, true, false);+ final Request request = this.request(location, true, false);final Response response = this.load(request, cachePolicy, blacklistType, agent);final DigestURL url = request.url();if (response == null) throw new IOException("no Response for url " + url);@@ -576,10 +576,10 @@ public final class LoaderDispatcher {// parse resourcetry {- Document[] documents = response.parse();- Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);+ final Document[] documents = response.parse();+ final Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);- String x_robots_tag = response.getResponseHeader().getXRobotsTag();+ final String x_robots_tag = response.getResponseHeader().getXRobotsTag();if (x_robots_tag.indexOf("noindex",0) >= 0) merged.setIndexingDenied(true);return merged;@@ -600,7 +600,7 @@ public final class LoaderDispatcher {public Document loadDocumentAsStream(final DigestURL location, final CacheStrategy cachePolicy,final BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException {// load resource- Request request = request(location, true, false);+ final Request request = this.request(location, true, false);final StreamResponse streamResponse = this.openInputStream(request, cachePolicy, blacklistType, agent);final Response response = streamResponse.getResponse();final DigestURL url = request.url();@@ -613,10 +613,10 @@ public final class LoaderDispatcher {// parse resourcetry {- Document[] documents = streamResponse.parse();- Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);+ final Document[] documents = streamResponse.parse();+ final Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);- String x_robots_tag = response.getResponseHeader().getXRobotsTag();+ final String x_robots_tag = response.getResponseHeader().getXRobotsTag();if (x_robots_tag.indexOf("noindex",0) >= 0) {merged.setIndexingDenied(true);}@@ -659,7 +659,7 @@ public final class LoaderDispatcher {public Document loadDocumentAsLimitedStream(final DigestURL location, final CacheStrategy cachePolicy,final BlacklistType blacklistType, final ClientIdentification.Agent agent, final int maxLinks, final long maxBytes) throws IOException {// load resource- Request request = request(location, true, false);+ final Request request = this.request(location, true, false);final StreamResponse streamResponse = this.openInputStream(request, cachePolicy, blacklistType, agent, -1);final Response response = streamResponse.getResponse();final DigestURL url = request.url();@@ -672,10 +672,10 @@ public final class LoaderDispatcher {// parse resourcetry {- Document[] documents = streamResponse.parseWithLimits(maxLinks, maxBytes);- Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);+ final Document[] documents = streamResponse.parseWithLimits(maxLinks, maxBytes);+ final Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);- String x_robots_tag = response.getResponseHeader().getXRobotsTag();+ final String x_robots_tag = response.getResponseHeader().getXRobotsTag();if (x_robots_tag.indexOf("noindex",0) >= 0) {merged.setIndexingDenied(true);}@@ -699,7 +699,7 @@ public final class LoaderDispatcher {BlacklistType blacklistType,final ClientIdentification.Agent agent,final int timezoneOffset) throws IOException {- final Response response = load(request(url, true, false), cacheStrategy, Integer.MAX_VALUE, blacklistType, agent);+ final Response response = this.load(this.request(url, true, false), cacheStrategy, Integer.MAX_VALUE, blacklistType, agent);if (response == null) throw new IOException("response == null");final ResponseHeader responseHeader = response.getResponseHeader();if (response.getContent() == null) throw new IOException("resource == null");@@ -769,7 +769,7 @@ public final class LoaderDispatcher {if (this.cache != null && this.cache.exists()) return;try {// load from the net- final Response response = load(request(this.url, false, true), this.cacheStrategy, this.maxFileSize, this.blacklistType, this.agent);+ final Response response = LoaderDispatcher.this.load(LoaderDispatcher.this.request(this.url, false, true), this.cacheStrategy, this.maxFileSize, this.blacklistType, this.agent);final byte[] b = response.getContent();if (this.cache != null) FileUtils.copy(b, this.cache);} catch (final MalformedURLException e) {} catch (final IOException e) {}diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.javaindex 073aec83c..edeaf7e67 100644--- a/source/net/yacy/server/serverSwitch.java+++ b/source/net/yacy/server/serverSwitch.java@@ -690,7 +690,7 @@ public class serverSwitch {netdef = netdef.trim();try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {final RequestHeader reqHeader = new RequestHeader();- reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);+ reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent());client.setHeader(reqHeader.entrySet());final byte[] data = client.GETbytes(uri,this.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),