Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

implemented (most of) https://github.com/yacy/yacy_search_server/issues/727 some refactorings done as well to support multiple user agents instead of only one; per request then is one of those user agents selected, not one random per session. Also: made sure that no robots.txt is loaded any more in case that the user agent is not a robot user agent.

Michael Peter Christenacb77bd68ab0
diff --git a/source/net/yacy/cora/protocol/ClientIdentification.java b/source/net/yacy/cora/protocol/ClientIdentification.java
index 8e3eba44b..25382f040 100644
--- a/source/net/yacy/cora/protocol/ClientIdentification.java
+++ b/source/net/yacy/cora/protocol/ClientIdentification.java
@@ -31,40 +31,55 @@ import java.util.concurrent.ConcurrentHashMap;
public class ClientIdentification {
- public static final int clientTimeoutInit = 10000;
+ public static final int clientTimeoutInit = 30000;
public static final int minimumLocalDeltaInit = 10; // the minimum time difference between access of the same local domain
public static final int minimumGlobalDeltaInit = 250; // the minimum time difference between access of the same global domain
public static class Agent {
- public final String userAgent; // the name that is send in http request to identify the agent
- public final String[] robotIDs; // the name that is used in robots.txt to identify the agent
- public final int minimumDelta; // the minimum delay between two accesses
- public final int clientTimeout;
- public Agent(final String userAgent, final String[] robotIDs, final int minimumDelta, final int clientTimeout) {
- this.userAgent = userAgent;
+ private final String[] userAgents; // the name that is send in http request to identify the agent
+ private final String[] robotIDs; // the name that is used in robots.txt to identify the agent; might be null or empty to signal that no robots.txt must be loaded (should only be use for browser agents)
+ private final int minimumDelta; // the minimum delay between two accesses
+ private final int clientTimeout;
+
+ public Agent(final String[] userAgents, final String[] robotIDs, final int minimumDelta, final int clientTimeout) {
+ this.userAgents = userAgents;
this.robotIDs = robotIDs;
this.minimumDelta = minimumDelta;
this.clientTimeout = clientTimeout;
}
+
+ public String userAgent() {
+ return this.userAgents.length == 1 ? this.userAgents[0] : this.userAgents[random.nextInt(this.userAgents.length)];
+ }
+
+ public boolean isRobot() {
+ return this.robotIDs != null && this.robotIDs.length > 0;
+ }
+
+ public String[] robotIDs() {
+ return this.robotIDs == null || this.robotIDs.length == 0 ? new String[] {"Mozilla"} : this.robotIDs;
+ }
+
+ public int minimumDelta() {
+ return this.minimumDelta;
+ }
+
+ public int clientTimeout() {
+ return this.clientTimeout;
+ }
}
- private final static String[] browserAgents = new String[]{ // fake browser user agents are NOT AVAILABLE IN P2P OPERATION, only on special customer configurations (commercial users demanded this, I personally think this is inadvisable)
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36",
- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:22.0) Gecko/20100101 Firefox/22.0",
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/536.30.1 (KHTML, like Gecko) Version/6.0.5 Safari/536.30.1",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36",
- "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36",
- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36",
- "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:22.0) Gecko/20100101 Firefox/22.0",
- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0",
- "Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/20100101 Firefox/22.0",
- "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20100101 Firefox/22.0",
- "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36",
- "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0"
- };
+ private final static String[] browserAgents = new String[]{
+ // see https://github.com/yacy/yacy_search_server/issues/727
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.3",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.3",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36"
+ };
private static final Random random = new Random(System.currentTimeMillis());
private static Map<String, Agent> agents = new ConcurrentHashMap<>();
public final static String yacyInternetCrawlerAgentName = "YaCy Internet (cautious)";
@@ -72,9 +87,10 @@ public class ClientIdentification {
public final static String yacyIntranetCrawlerAgentName = "YaCy Intranet (greedy)";
public static Agent yacyIntranetCrawlerAgent = null; // defined later in static
public final static String googleAgentName = "Googlebot";
- public final static Agent googleAgentAgent = new Agent("Googlebot/2.1 (+http://www.google.com/bot.html)", new String[]{"Googlebot", "Googlebot-Mobile"}, minimumGlobalDeltaInit / 10, clientTimeoutInit);
+ //public final static Agent googleAgentAgent = new Agent("Googlebot/2.1 (+http://www.google.com/bot.html)", new String[]{"Googlebot", "Googlebot-Mobile"}, minimumGlobalDeltaInit / 10, clientTimeoutInit);
+ public final static Agent googleAgentAgent = new Agent(new String[]{"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}, new String[]{"Googlebot", "Googlebot-Mobile"}, minimumGlobalDeltaInit / 10, clientTimeoutInit); // see https://github.com/yacy/yacy_search_server/issues/727
public final static String yacyProxyAgentName = "YaCyProxy";
- public final static Agent yacyProxyAgent = new Agent("yacy - this is a proxy access through YaCy from a browser, not a robot (the yacy bot user agent is 'yacybot')", new String[]{"yacy"}, minimumGlobalDeltaInit, clientTimeoutInit);
+ public final static Agent yacyProxyAgent = new Agent(new String[]{"yacy - this is a proxy access through YaCy from a browser, not a robot (the yacy bot user agent is 'yacybot')"}, new String[]{"yacy"}, minimumGlobalDeltaInit, clientTimeoutInit);
public final static String customAgentName = "Custom Agent";
public final static String browserAgentName = "Random Browser";
public static Agent browserAgent;
@@ -88,7 +104,7 @@ public class ClientIdentification {
static {
generateYaCyBot("new");
- browserAgent = new Agent(browserAgents[random.nextInt(browserAgents.length)], new String[]{"Mozilla"}, minimumLocalDeltaInit, clientTimeoutInit);
+ browserAgent = new Agent(browserAgents, new String[0], minimumGlobalDeltaInit, clientTimeoutInit);
agents.put(googleAgentName, googleAgentAgent);
agents.put(browserAgentName, browserAgent);
agents.put(yacyProxyAgentName, yacyProxyAgent);
@@ -101,8 +117,8 @@ public class ClientIdentification {
*/
public static void generateYaCyBot(final String addinfo) {
final String agentString = "yacybot (" + addinfo + "; " + yacySystem + ") https://yacy.net/bot.html";
- yacyInternetCrawlerAgent = new Agent(agentString, new String[]{"yacybot"}, minimumGlobalDeltaInit, clientTimeoutInit);
- yacyIntranetCrawlerAgent = new Agent(agentString, new String[]{"yacybot"}, minimumLocalDeltaInit, clientTimeoutInit); // must have the same userAgent String as the web crawler because this is also used for snippets
+ yacyInternetCrawlerAgent = new Agent(new String[]{agentString}, new String[]{"yacybot"}, minimumGlobalDeltaInit, clientTimeoutInit);
+ yacyIntranetCrawlerAgent = new Agent(new String[]{agentString}, new String[]{"yacybot"}, minimumLocalDeltaInit, clientTimeoutInit); // must have the same userAgent String as the web crawler because this is also used for snippets
agents.put(yacyInternetCrawlerAgentName, yacyInternetCrawlerAgent);
agents.put(yacyIntranetCrawlerAgentName, yacyIntranetCrawlerAgent);
}
@@ -110,7 +126,7 @@ public class ClientIdentification {
public static void generateCustomBot(final String name, final String string, final int minimumdelta, final int clienttimeout) {
if (name.toLowerCase().indexOf("yacy") >= 0 || string.toLowerCase().indexOf("yacy") >= 0) return; // don't allow 'yacy' in custom bot strings
final String agentString = string.replace("$$SYSTEM$$", yacySystem.replace("java", "O"));
- agents.put(customAgentName, new Agent(agentString, new String[]{name}, minimumdelta, clienttimeout));
+ agents.put(customAgentName, new Agent(new String[]{agentString}, new String[]{name}, minimumdelta, clienttimeout));
}
/**
diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java
index 015a13c36..3ff85a567 100644
--- a/source/net/yacy/cora/protocol/http/HTTPClient.java
+++ b/source/net/yacy/cora/protocol/http/HTTPClient.java
@@ -181,18 +181,18 @@ public class HTTPClient implements Closeable {
public HTTPClient(final ClientIdentification.Agent agent) {
super();
- this.timeout = agent.clientTimeout;
+ this.timeout = agent.clientTimeout();
this.clientBuilder = initClientBuilder();
- this.clientBuilder.setUserAgent(agent.userAgent);
+ this.clientBuilder.setUserAgent(agent.userAgent());
this.reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG);
- this.setTimout(agent.clientTimeout);
+ this.setTimout(agent.clientTimeout());
}
public HTTPClient(final ClientIdentification.Agent agent, final int timeout) {
super();
this.timeout = timeout;
this.clientBuilder = initClientBuilder();
- this.clientBuilder.setUserAgent(agent.userAgent);
+ this.clientBuilder.setUserAgent(agent.userAgent());
this.reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG);
this.setTimout(timeout);
}
@@ -222,7 +222,7 @@ public class HTTPClient implements Closeable {
builder.setDefaultRequestConfig(DFLTREQUESTCONFIG);
// UserAgent
- builder.setUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
+ builder.setUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent());
// remove retries; we expect connections to fail; therefore we should not retry
//builder.disableAutomaticRetries();
@@ -360,7 +360,7 @@ public class HTTPClient implements Closeable {
* @param userAgent
*/
public void setUserAgent(final ClientIdentification.Agent agent) {
- this.clientBuilder.setUserAgent(agent.userAgent);
+ this.clientBuilder.setUserAgent(agent.userAgent());
}
/**
diff --git a/source/net/yacy/cora/util/Html2Image.java b/source/net/yacy/cora/util/Html2Image.java
index 83efe559b..ccd44d31a 100644
--- a/source/net/yacy/cora/util/Html2Image.java
+++ b/source/net/yacy/cora/util/Html2Image.java
@@ -26,8 +26,6 @@ import java.awt.Graphics;
import java.awt.Image;
import java.awt.MediaTracker;
import java.awt.image.BufferedImage;
-import java.beans.PropertyChangeEvent;
-import java.beans.PropertyChangeListener;
import java.io.File;
import java.io.IOException;
import java.util.List;
@@ -463,10 +461,7 @@ public class Html2Image {
};
htmlPane.setEditorKitForContentType("text/html", kit);
htmlPane.setContentType("text/html");
- htmlPane.addPropertyChangeListener(new PropertyChangeListener() {
- @Override
- public void propertyChange(final PropertyChangeEvent evt) {
- }
+ htmlPane.addPropertyChangeListener(evt -> {
});
// load the page
@@ -524,7 +519,7 @@ public class Html2Image {
exitStatus = 1;
return;
}
- if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,
+ if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent(),
"en-us,en;q=0.5", targetPdfFile, 30)) {
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {
diff --git a/source/net/yacy/crawler/HostBalancer.java b/source/net/yacy/crawler/HostBalancer.java
index c9b8133cf..afec24e3e 100644
--- a/source/net/yacy/crawler/HostBalancer.java
+++ b/source/net/yacy/crawler/HostBalancer.java
@@ -67,7 +67,7 @@ public class HostBalancer implements Balancer {
private final static ConcurrentLog log = new ConcurrentLog("HostBalancer");
public final static HandleMap depthCache = new RowHandleMap(Word.commonHashLength, Word.commonHashOrder, 2, 8 * 1024 * 1024, "HostBalancer.DepthCache");
-
+ public final static ClientIdentification.Agent unknwonAgentDefault = ClientIdentification.browserAgent;
private final File hostsPath;
private final boolean exceed134217727;
private final ConcurrentHashMap<String, HostQueue> queues;
@@ -347,7 +347,8 @@ public class HostBalancer implements Balancer {
final String hosthash = i.next();
final HostQueue hq = this.queues.get(hosthash);
if (hq == null) {removehosts.add(hosthash); i.remove(); continue smallstacks;}
- final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hosthash, robots, ClientIdentification.yacyInternetCrawlerAgent);
+
+ final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hosthash, robots, unknwonAgentDefault);
if (delta == Integer.MIN_VALUE) {
// never-crawled hosts; we do not want to have too many of them in here. Loading new hosts means: waiting for robots.txt to load
freshhosts.add(hosthash);
@@ -404,7 +405,7 @@ public class HostBalancer implements Balancer {
mixedstrategy: for (final String h: this.roundRobinHostHashes) {
final HostQueue hq = this.queues.get(h);
if (hq != null) {
- int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), h, robots, ClientIdentification.yacyInternetCrawlerAgent) / 200;
+ int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), h, robots, unknwonAgentDefault) / 200;
if (delta < 0) delta = 0;
List<String> queueHashes = fastTree.get(delta);
if (queueHashes == null) {
@@ -499,7 +500,7 @@ public class HostBalancer implements Balancer {
final String s = i.next();
final HostQueue hq = this.queues.get(s);
if (hq == null) {i.remove(); continue protectcheck;}
- final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), s, robots, ClientIdentification.yacyInternetCrawlerAgent);
+ final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), s, robots, unknwonAgentDefault);
if (delta >= 0) {i.remove();}
}
}
@@ -529,7 +530,7 @@ public class HostBalancer implements Balancer {
@SuppressWarnings("unchecked")
final Iterator<Request>[] hostIterator = (Iterator<Request>[]) Array.newInstance(Iterator.class, 1);
hostIterator[0] = null;
- return new Iterator<Request>() {
+ return new Iterator<>() {
@Override
public boolean hasNext() {
return hostsIterator.hasNext() || (hostIterator[0] != null && hostIterator[0].hasNext());
@@ -560,7 +561,7 @@ public class HostBalancer implements Balancer {
public Map<String, Integer[]> getDomainStackHosts(final RobotsTxt robots) {
final Map<String, Integer[]> map = new TreeMap<>(); // we use a tree map to get a stable ordering
for (final HostQueue hq: this.queues.values()) {
- final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hq.getHostHash(), robots, ClientIdentification.yacyInternetCrawlerAgent);
+ final int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hq.getHostHash(), robots, unknwonAgentDefault);
map.put(hq.getHost() + ":" + hq.getPort(), new Integer[]{hq.size(), delta});
}
return map;
diff --git a/source/net/yacy/crawler/data/CrawlQueues.java b/source/net/yacy/crawler/data/CrawlQueues.java
index 38c8e8a2c..fd0fd971e 100644
--- a/source/net/yacy/crawler/data/CrawlQueues.java
+++ b/source/net/yacy/crawler/data/CrawlQueues.java
@@ -72,7 +72,7 @@ import net.yacy.search.index.ErrorCache;
import net.yacy.search.index.ErrorCacheFiller;
public class CrawlQueues {
-
+
private final static Request POISON_REQUEST = new Request();
private final static ConcurrentLog log = new ConcurrentLog("CRAWLER");
@@ -83,7 +83,7 @@ public class CrawlQueues {
public NoticedURL noticeURL;
public ErrorCache errorURL;
-
+
/** URLs pulled by remote peers in order to crawl them for us */
public Map<String, DigestURL> delegatedURL;
@@ -91,9 +91,9 @@ public class CrawlQueues {
this.sb = sb;
final int maxWorkers = (int) sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10);
this.worker = new Loader[maxWorkers];
- /* We initialize workerQueue with the same capacity as worker array, because this same queue
+ /* We initialize workerQueue with the same capacity as worker array, because this same queue
* will be used to send POISON_REQUEST items consumed by all eventually running workers in the close() function*/
- this.workerQueue = new ArrayBlockingQueue<Request>(maxWorkers);
+ this.workerQueue = new ArrayBlockingQueue<>(maxWorkers);
this.remoteCrawlProviderHashes = null;
// start crawling management
@@ -107,9 +107,9 @@ public class CrawlQueues {
}
public void initRemoteCrawlQueues () {
- if (this.remoteCrawlProviderHashes == null) this.remoteCrawlProviderHashes = new ArrayList<String>();
+ if (this.remoteCrawlProviderHashes == null) this.remoteCrawlProviderHashes = new ArrayList<>();
if (this.delegatedURL == null) {
- this.delegatedURL = new ConcurrentHashMap<String, DigestURL>();
+ this.delegatedURL = new ConcurrentHashMap<>();
log.config("Finished Startup of Crawling Management");
}
}
@@ -125,10 +125,10 @@ public class CrawlQueues {
this.errorURL.clearCache();
/* Concurrently refill the error cache with recent errors from the index */
new ErrorCacheFiller(this.sb, this.errorURL).start();
-
+
if (this.remoteCrawlProviderHashes != null) this.remoteCrawlProviderHashes.clear();
this.noticeURL.close();
- this.noticeURL = new NoticedURL(newQueuePath, sb.getConfigInt("crawler.onDemandLimit", 1000), this.sb.exceed134217727);
+ this.noticeURL = new NoticedURL(newQueuePath, this.sb.getConfigInt("crawler.onDemandLimit", 1000), this.sb.exceed134217727);
if (this.delegatedURL != null) this.delegatedURL.clear();
}
@@ -139,7 +139,7 @@ public class CrawlQueues {
this.workerQueue.clear();
// wait for all workers to finish
for (int i = 0; i < this.workerQueue.remainingCapacity(); i++) {
- /* We use the non-blocking offer() function instead of the blocking put() in the unlikely eventual case another thread
+ /* We use the non-blocking offer() function instead of the blocking put() in the unlikely eventual case another thread
* added an element to workerQueue during this loop.*/
try {
this.workerQueue.offer(POISON_REQUEST, 1, TimeUnit.SECONDS);
@@ -181,14 +181,14 @@ public class CrawlQueues {
//if (this.noticeURL.existsInStack(hash)) {
// return HarvestProcess.CRAWLER;
//} // this is disabled because it prevents proper crawling of smb shares. The cause is unknown
- for (final Request request: activeWorkerEntries().values()) {
+ for (final Request request: this.activeWorkerEntries().values()) {
if (Base64Order.enhancedCoder.equal(request.url().hash(), hash)) {
return HarvestProcess.WORKER;
}
}
return null;
}
-
+
/**
* count the number of same host names in the worker
* @param host
@@ -197,7 +197,7 @@ public class CrawlQueues {
public int hostcount(final String host) {
if (host == null || host.length() == 0) return 0;
int c = 0;
- for (final DigestURL url: activeWorkerEntries().keySet()) {
+ for (final DigestURL url: this.activeWorkerEntries().keySet()) {
if (host.equals(url.getHost())) {
c++;
}
@@ -212,12 +212,12 @@ public class CrawlQueues {
this.delegatedURL.remove(ASCII.String(hash));
}
}
-
+
public int removeHosts(final Set<String> hosthashes) {
return this.noticeURL.removeByHostHash(hosthashes);
//this.delegatedURL.remove(hash);
}
-
+
public DigestURL getURL(final byte[] urlhash) {
assert urlhash != null;
if (urlhash == null || urlhash.length == 0) {
@@ -229,7 +229,7 @@ public class CrawlQueues {
return u;
}
}
- for (final DigestURL url: activeWorkerEntries().keySet()) {
+ for (final DigestURL url: this.activeWorkerEntries().keySet()) {
if (Base64Order.enhancedCoder.equal(url.hash(), urlhash)) {
return url;
}
@@ -249,10 +249,10 @@ public class CrawlQueues {
this.errorURL.clearStack();
}
}
-
+
public Map<DigestURL, Request> activeWorkerEntries() {
synchronized (this.worker) {
- Map<DigestURL, Request> map = new HashMap<DigestURL, Request>();
+ Map<DigestURL, Request> map = new HashMap<>();
for (final Loader w: this.worker) {
if (w != null) {
Request r = w.loading();
@@ -271,19 +271,19 @@ public class CrawlQueues {
final boolean robinsonPrivateCase = (this.sb.isRobinsonMode() &&
!this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER));
- if ((robinsonPrivateCase || coreCrawlJobSize() <= 20) && limitCrawlJobSize() > 0) {
+ if ((robinsonPrivateCase || this.coreCrawlJobSize() <= 20) && this.limitCrawlJobSize() > 0) {
// move some tasks to the core crawl job so we have something to do
- final int toshift = Math.min(10, limitCrawlJobSize()); // this cannot be a big number because the balancer makes a forced waiting if it cannot balance
+ final int toshift = Math.min(10, this.limitCrawlJobSize()); // this cannot be a big number because the balancer makes a forced waiting if it cannot balance
for (int i = 0; i < toshift; i++) {
this.noticeURL.shift(NoticedURL.StackType.GLOBAL, NoticedURL.StackType.LOCAL, this.sb.crawler, this.sb.robots);
}
- CrawlQueues.log.info("shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()=" + coreCrawlJobSize() +
- ", limitCrawlJobSize()=" + limitCrawlJobSize() + ", cluster.mode=" + this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "") +
+ CrawlQueues.log.info("shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()=" + this.coreCrawlJobSize() +
+ ", limitCrawlJobSize()=" + this.limitCrawlJobSize() + ", cluster.mode=" + this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "") +
", robinsonMode=" + ((this.sb.isRobinsonMode()) ? "on" : "off"));
}
- final String queueCheckCore = loadIsPossible(NoticedURL.StackType.LOCAL);
- final String queueCheckNoload = loadIsPossible(NoticedURL.StackType.NOLOAD);
+ final String queueCheckCore = this.loadIsPossible(NoticedURL.StackType.LOCAL);
+ final String queueCheckNoload = this.loadIsPossible(NoticedURL.StackType.NOLOAD);
if (queueCheckCore != null && queueCheckNoload != null) {
if (CrawlQueues.log.isFine()) {
CrawlQueues.log.fine("omitting de-queue/local: " + queueCheckCore + ":" + queueCheckNoload);
@@ -291,7 +291,7 @@ public class CrawlQueues {
return false;
}
- if (isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
+ if (this.isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
if (CrawlQueues.log.isFine()) {
CrawlQueues.log.fine("omitting de-queue/local: paused");
}
@@ -304,7 +304,7 @@ public class CrawlQueues {
final String stats = "LOCALCRAWL[" +
this.noticeURL.stackSize(NoticedURL.StackType.NOLOAD) + ", " +
this.noticeURL.stackSize(NoticedURL.StackType.LOCAL) + ", " +
- this.noticeURL.stackSize(NoticedURL.StackType.GLOBAL) +
+ this.noticeURL.stackSize(NoticedURL.StackType.GLOBAL) +
", " + this.noticeURL.stackSize(NoticedURL.StackType.REMOTE) + "]";
try {
if (!this.noticeURL.isEmpty(NoticedURL.StackType.NOLOAD)) {
@@ -338,7 +338,7 @@ public class CrawlQueues {
CrawlQueues.log.severe(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
return true;
}
- load(urlEntry, stats);
+ this.load(urlEntry, stats);
return true;
} catch (final IOException e) {
CrawlQueues.log.severe(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);
@@ -381,9 +381,9 @@ public class CrawlQueues {
if (urlEntry == null || urlEntry.url() == null) {
CrawlQueues.log.info(stats + ": urlEntry = null");
} else {
- if (!activeWorkerEntries().containsKey(urlEntry.url())) {
+ if (!this.activeWorkerEntries().containsKey(urlEntry.url())) {
try {
- ensureLoaderRunning();
+ this.ensureLoaderRunning();
this.workerQueue.put(urlEntry);
} catch (InterruptedException e) {
ConcurrentLog.logException(e);
@@ -471,14 +471,14 @@ public class CrawlQueues {
return false;
}
- if (remoteTriggeredCrawlJobSize() > 200) {
+ if (this.remoteTriggeredCrawlJobSize() > 200) {
if (CrawlQueues.log.isFine()) {
CrawlQueues.log.fine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing");
}
return false;
}
- if (coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) {
+ if (this.coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) {
if (CrawlQueues.log.isFine()) {
CrawlQueues.log.fine("remoteCrawlLoaderJob: a local crawl is running, omitting processing");
}
@@ -525,12 +525,12 @@ public class CrawlQueues {
}
// we know a peer which should provide remote crawl entries. load them now.
- final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
+ final boolean preferHttps = this.sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
final RSSFeed feed = Protocol.queryRemoteCrawlURLs(this.sb.peers, seed, 60, 10000, preferHttps);
if (feed == null || feed.isEmpty()) {
// try again and ask another peer
- return remoteCrawlLoaderJob();
+ return this.remoteCrawlLoaderJob();
}
// parse the rss
@@ -573,24 +573,24 @@ public class CrawlQueues {
}
return true;
}
-
+
public boolean autocrawlJob() {
if (!this.sb.getConfigBool(SwitchboardConstants.AUTOCRAWL, false)) {
return false;
}
-
- if (isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
+
+ if (this.isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
return false;
}
-
- if (coreCrawlJobSize() > 200) {
+
+ if (this.coreCrawlJobSize() > 200) {
return false;
}
-
+
String rows = this.sb.getConfig(SwitchboardConstants.AUTOCRAWL_ROWS, "100");
-
+
String dateQuery = String.format("load_date_dt:[* TO NOW-%sDAY]", this.sb.getConfig(SwitchboardConstants.AUTOCRAWL_DAYS, "1"));
-
+
final SolrQuery query = new SolrQuery();
query.add("group", "true");
query.add("group.field", "host_s");
@@ -601,10 +601,10 @@ public class CrawlQueues {
query.setFields("host_s,url_protocol_s");
query.addSort("load_date_dt", SolrQuery.ORDER.asc);
query.addFilterQuery(dateQuery);
-
+
try {
- QueryResponse resp = sb.index.fulltext().getDefaultConnector().getResponseByParams(query);
-
+ QueryResponse resp = this.sb.index.fulltext().getDefaultConnector().getResponseByParams(query);
+
int i = 0;
int deepRatio = Integer.parseInt(this.sb.getConfig(SwitchboardConstants.AUTOCRAWL_RATIO, "50"));
for (SolrDocument doc: resp.getResults()) {
@@ -619,7 +619,7 @@ public class CrawlQueues {
DigestURL url;
if (doc.getFieldValue("url_protocol_s") == null || doc.getFieldValue("host_s") == null) {
//Skip this document if either of these values is null.
- continue;
+ continue;
}
final String u = doc.getFieldValue("url_protocol_s").toString() + "://" + doc.getFieldValue("host_s").toString();
try {
@@ -643,13 +643,13 @@ public class CrawlQueues {
CrawlQueues.log.warn("autocrawl: Rejected URL '" + urlToString(url) + "': " + urlRejectReason);
}
}
-
+
} catch (SolrException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
-
+
return true;
}
@@ -678,7 +678,7 @@ public class CrawlQueues {
// do nothing if either there are private processes to be done
// or there is no global crawl on the stack
- final String queueCheck = loadIsPossible(NoticedURL.StackType.REMOTE);
+ final String queueCheck = this.loadIsPossible(NoticedURL.StackType.REMOTE);
if (queueCheck != null) {
if (CrawlQueues.log.isFinest()) {
CrawlQueues.log.finest("omitting de-queue/remote: " + queueCheck);
@@ -686,7 +686,7 @@ public class CrawlQueues {
return false;
}
- if (isPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
+ if (this.isPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
if (CrawlQueues.log.isFinest()) {
CrawlQueues.log.finest("omitting de-queue/remote: paused");
}
@@ -699,7 +699,7 @@ public class CrawlQueues {
try {
final Request urlEntry = this.noticeURL.pop(NoticedURL.StackType.REMOTE, true, this.sb.crawler, this.sb.robots);
if (urlEntry == null) return false;
- load(urlEntry, stats);
+ this.load(urlEntry, stats);
return true;
} catch (final IOException e) {
CrawlQueues.log.severe(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);
@@ -721,84 +721,85 @@ public class CrawlQueues {
if (this.worker[i].loading() == null) return;
}
}
-
+
private final class Loader extends Thread {
private Request request = null;
private Loader() {
}
-
+
public Request loading() {
- return request;
+ return this.request;
}
@Override
public void run() {
this.setPriority(Thread.MIN_PRIORITY); // http requests from the crawler should not cause that other functions work worse
try {
- while ((request = CrawlQueues.this.workerQueue.poll(10, TimeUnit.SECONDS)) != POISON_REQUEST) {
- if (request == null) break; // we run this only for a specific time and then let the process die to clear up resources
- request.setStatus("worker-initialized", WorkflowJob.STATUS_INITIATED);
- this.setName("CrawlQueues.Loader(" + request.url().toNormalform(false) + ")");
- CrawlProfile profile = CrawlQueues.this.sb.crawler.get(UTF8.getBytes(request.profileHandle()));
+ while ((this.request = CrawlQueues.this.workerQueue.poll(10, TimeUnit.SECONDS)) != POISON_REQUEST) {
+ if (this.request == null) break; // we run this only for a specific time and then let the process die to clear up resources
+ this.request.setStatus("worker-initialized", WorkflowJob.STATUS_INITIATED);
+ this.setName("CrawlQueues.Loader(" + this.request.url().toNormalform(false) + ")");
+ CrawlProfile profile = CrawlQueues.this.sb.crawler.get(UTF8.getBytes(this.request.profileHandle()));
try {
// checking robots.txt for http(s) resources
- request.setStatus("worker-checkingrobots", WorkflowJob.STATUS_STARTED);
+ this.request.setStatus("worker-checkingrobots", WorkflowJob.STATUS_STARTED);
RobotsTxtEntry robotsEntry;
- if ((request.url().getProtocol().equals("http") || request.url().getProtocol().equals("https")) &&
- (robotsEntry = CrawlQueues.this.sb.robots.getEntry(request.url(), profile.getAgent())) != null &&
- robotsEntry.isDisallowed(request.url())) {
+ if ((this.request.url().getProtocol().equals("http") || this.request.url().getProtocol().equals("https")) &&
+ profile.getAgent().isRobot() &&
+ (robotsEntry = CrawlQueues.this.sb.robots.getEntry(this.request.url(), profile.getAgent())) != null &&
+ robotsEntry.isDisallowed(this.request.url())) {
//if (log.isFine()) log.logFine("Crawling of URL '" + request.url().toString() + "' disallowed by robots.txt.");
- CrawlQueues.this.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_ROBOTS_RULE, "denied by robots.txt", -1);
- request.setStatus("worker-disallowed", WorkflowJob.STATUS_FINISHED);
+ CrawlQueues.this.errorURL.push(this.request.url(), this.request.depth(), profile, FailCategory.FINAL_ROBOTS_RULE, "denied by robots.txt", -1);
+ this.request.setStatus("worker-disallowed", WorkflowJob.STATUS_FINISHED);
} else {
// starting a load from the internet
- request.setStatus("worker-loading", WorkflowJob.STATUS_RUNNING);
+ this.request.setStatus("worker-loading", WorkflowJob.STATUS_RUNNING);
String error = null;
-
+
// load a resource and push queue entry to switchboard queue
// returns null if everything went fine, a fail reason string if a problem occurred
try {
- request.setStatus("loading", WorkflowJob.STATUS_RUNNING);
- final Response response = CrawlQueues.this.sb.loader.load(request, profile == null ? CacheStrategy.IFEXIST : profile.cacheStrategy(), BlacklistType.CRAWLER, profile.getAgent());
+ this.request.setStatus("loading", WorkflowJob.STATUS_RUNNING);
+ final Response response = CrawlQueues.this.sb.loader.load(this.request, profile == null ? CacheStrategy.IFEXIST : profile.cacheStrategy(), BlacklistType.CRAWLER, profile.getAgent());
if (response == null) {
- request.setStatus("error", WorkflowJob.STATUS_FINISHED);
+ this.request.setStatus("error", WorkflowJob.STATUS_FINISHED);
if (CrawlQueues.log.isFine()) {
- CrawlQueues.log.fine("problem loading " + request.url().toString() + ": no content (possibly caused by cache policy)");
+ CrawlQueues.log.fine("problem loading " + this.request.url().toString() + ": no content (possibly caused by cache policy)");
}
error = "no content (possibly caused by cache policy)";
} else {
- request.setStatus("loaded", WorkflowJob.STATUS_RUNNING);
+ this.request.setStatus("loaded", WorkflowJob.STATUS_RUNNING);
final String storedFailMessage = CrawlQueues.this.sb.toIndexer(response);
- request.setStatus("enqueued-" + ((storedFailMessage == null) ? "ok" : "fail"), WorkflowJob.STATUS_FINISHED);
+ this.request.setStatus("enqueued-" + ((storedFailMessage == null) ? "ok" : "fail"), WorkflowJob.STATUS_FINISHED);
error = (storedFailMessage == null) ? null : "not enqueued to indexer: " + storedFailMessage;
}
} catch (final IOException e) {
- request.setStatus("error", WorkflowJob.STATUS_FINISHED);
+ this.request.setStatus("error", WorkflowJob.STATUS_FINISHED);
if (CrawlQueues.log.isFine()) {
- CrawlQueues.log.fine("problem loading " + request.url().toString() + ": " + e.getMessage());
+ CrawlQueues.log.fine("problem loading " + this.request.url().toString() + ": " + e.getMessage());
}
error = "load error - " + e.getMessage();
}
-
+
if (error != null) {
if (error.endsWith("$")) {
// the "$" mark at the end of the error message means, that the error was already pushed to the error-db by the reporting method
// thus we only push this message if we don't have that mark
error = error.substring(0, error.length() - 1).trim();
} else {
- CrawlQueues.this.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "cannot load: " + error, -1);
+ CrawlQueues.this.errorURL.push(this.request.url(), this.request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "cannot load: " + error, -1);
}
- request.setStatus("worker-error", WorkflowJob.STATUS_FINISHED);
+ this.request.setStatus("worker-error", WorkflowJob.STATUS_FINISHED);
} else {
- request.setStatus("worker-processed", WorkflowJob.STATUS_FINISHED);
+ this.request.setStatus("worker-processed", WorkflowJob.STATUS_FINISHED);
}
}
} catch (final Exception e) {
- CrawlQueues.this.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, e.getMessage() + " - in worker", -1);
- request.setStatus("worker-exception", WorkflowJob.STATUS_FINISHED);
+ CrawlQueues.this.errorURL.push(this.request.url(), this.request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, e.getMessage() + " - in worker", -1);
+ this.request.setStatus("worker-exception", WorkflowJob.STATUS_FINISHED);
} finally {
- request = null;
+ this.request = null;
this.setName("CrawlQueues.Loader(WAITING)");
}
profile = null;
diff --git a/source/net/yacy/crawler/data/Latency.java b/source/net/yacy/crawler/data/Latency.java
index 07c3faba2..8aab14514 100644
--- a/source/net/yacy/crawler/data/Latency.java
+++ b/source/net/yacy/crawler/data/Latency.java
@@ -122,6 +122,7 @@ public class Latency {
* @return the waiting time in milliseconds; 0 if not known; -1 if host gives us special rights
*/
public static int waitingRobots(final MultiProtocolURL url, final RobotsTxt robots, final ClientIdentification.Agent agent) {
+ if (!agent.isRobot()) return agent.minimumDelta();
int robotsDelay;
final RobotsTxtEntry robotsEntry = robots.getEntry(url, agent);
robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis();
@@ -130,6 +131,7 @@ public class Latency {
}
private static int waitingRobots(final String hostport, final RobotsTxt robots, final ClientIdentification.Agent agent, final boolean fetchOnlineIfNotAvailableOrNotFresh) {
+ if (!agent.isRobot()) return agent.minimumDelta();
int robotsDelay;
final RobotsTxtEntry robotsEntry = robots.getEntry(hostport, agent, fetchOnlineIfNotAvailableOrNotFresh);
robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis();
@@ -154,7 +156,7 @@ public class Latency {
if (host == null) return Integer.MIN_VALUE; // no delay if host is new; use Integer because there is a cast to int somewhere
// find the minimum waiting time based on the network domain (local or global)
- int waiting = agent.minimumDelta;
+ int waiting = agent.minimumDelta();
// if we have accessed the domain many times, get slower (the flux factor)
waiting += host.flux(waiting);
@@ -199,7 +201,7 @@ public class Latency {
// find the minimum waiting time based on the network domain (local or global)
final boolean local = url.isLocal();
- int waiting = agent.minimumDelta;
+ int waiting = agent.minimumDelta();
// if we have accessed the domain many times, get slower (the flux factor)
if (!local) waiting += host.flux(waiting);
@@ -232,7 +234,7 @@ public class Latency {
final StringBuilder s = new StringBuilder(50);
// find the minimum waiting time based on the network domain (local or global)
- int waiting = agent.minimumDelta;
+ int waiting = agent.minimumDelta();
s.append("minimumDelta = ").append(waiting);
// if we have accessed the domain many times, get slower (the flux factor)
diff --git a/source/net/yacy/crawler/data/Transactions.java b/source/net/yacy/crawler/data/Transactions.java
index 499dae7da..016cd06c6 100644
--- a/source/net/yacy/crawler/data/Transactions.java
+++ b/source/net/yacy/crawler/data/Transactions.java
@@ -1,6 +1,6 @@
/**
* Transactions
- * SPDX-FileCopyrightText: 2014 Michael Peter Christen <[email protected])>
+ * SPDX-FileCopyrightText: 2014 Michael Peter Christen <[email protected])>
* SPDX-License-Identifier: GPL-2.0-or-later
* First released 08.12.2014 at https://yacy.net
*
@@ -56,7 +56,7 @@ import net.yacy.search.schema.CollectionSchema;
*
*/
public class Transactions {
-
+
private final static String XML_PREFIX = "<response>\n<!--\n";
private final static char[] WHITESPACE = new char[132];
//private final static int WHITESPACE_START = XML_PREFIX.length();
@@ -65,14 +65,14 @@ public class Transactions {
private static Snapshots inventory = null, archive = null;
private static ExecutorService executor = Executors.newCachedThreadPool();
private static AtomicInteger executorRunning = new AtomicInteger(0);
-
+
/** the maximum to wait for each wkhtmltopdf call when rendering PDF snapshots */
private static long wkhtmltopdfTimeout = 30;
-
+
static {
for (int i = 0; i < WHITESPACE.length; i++) WHITESPACE[i] = 32;
}
-
+
public static enum State {
INVENTORY("inventory"), ARCHIVE("archive"), ANY(null);
public String dirname;
@@ -80,7 +80,7 @@ public class Transactions {
this.dirname = dirname;
}
}
-
+
/**
* @param dir the parent directory of inventory and archive snapshots.
* @param wkhtmltopdfSecondsTimeout the maximum to wait for each wkhtmltopdf call when rendering PDF snapshots
@@ -94,7 +94,7 @@ public class Transactions {
archive = new Snapshots(archiveDir);
wkhtmltopdfTimeout = wkhtmltopdfSecondsTimeout;
}
-
+
public static synchronized void migrateIPV6Snapshots() {
executor.shutdown();
try {
@@ -122,7 +122,7 @@ public class Transactions {
default : Revisions a = inventory.getRevisions(urlhash); return a == null ? archive.getRevisions(urlhash) : a;
}
}
-
+
/**
* get a list of <host>.<port> names in the snapshot directory
* @return the list of the given state. if the state is ALL or unknown, all lists are combined
@@ -134,12 +134,12 @@ public class Transactions {
default : Set<String> a = inventory.listHosts(); a.addAll(archive.listHosts()); return a;
}
}
-
+
/**
* list the snapshots for a given host name
* @param hostport the <host>.<port> identifier for the domain (with the same format as applied by the Snapshots.pathToHostPortDir() function).
* @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
* @return a map with a set for each depth in the domain of the host name
*/
public static TreeMap<Integer, Collection<Revisions>> listIDs(final String hostport, final int depth, final State state) {
@@ -149,12 +149,12 @@ public class Transactions {
default : TreeMap<Integer, Collection<Revisions>> a = inventory.listIDs(hostport, depth); a.putAll(archive.listIDs(hostport, depth)); return a;
}
}
-
+
/**
* get the number of snapshots for the given host name
* @param hostport the <host>.<port> identifier for the domain
* @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
* @return a count, the total number of documents for the domain and depth
*/
public static int listIDsSize(final String hostport, final int depth, final State state) {
@@ -164,7 +164,7 @@ public class Transactions {
default : return inventory.listIDsSize(hostport, depth) + archive.listIDsSize(hostport, depth);
}
}
-
+
public static boolean store(final SolrInputDocument doc, final boolean concurrency, final boolean loadImage, final boolean replaceOld, final String proxy, final String acceptLanguage) {
// GET METADATA FROM DOC
@@ -180,7 +180,7 @@ public class Transactions {
ConcurrentLog.logException(e);
return false;
}
-
+
boolean success = loadImage ? store(url, date, depth, concurrency, replaceOld, proxy, acceptLanguage) : true;
if (success) {
// STORE METADATA FOR THE IMAGE
@@ -203,15 +203,15 @@ public class Transactions {
success = false;
}
if(success) {
- Transactions.announceStorage(url, depth, date, State.INVENTORY);
+ Transactions.announceStorage(url, depth, date, State.INVENTORY);
}
}
-
+
}
-
+
return success;
}
-
+
public static boolean store(final DigestURL url, final Date date, final int depth, final boolean concurrency, final boolean replaceOld, final String proxy, final String acceptLanguage) {
@@ -222,12 +222,12 @@ public class Transactions {
oldPath.delete();
}
}
-
+
// STORE METADATA FOR THE IMAGE
File metadataPath = Transactions.definePath(url, depth, date, "xml", Transactions.State.INVENTORY);
metadataPath.getParentFile().mkdirs();
boolean success = true;
-
+
// STORE AN IMAGE
final String urls = url.toNormalform(true);
final File pdfPath = Transactions.definePath(url, depth, date, "pdf", Transactions.State.INVENTORY);
@@ -237,7 +237,7 @@ public class Transactions {
public void run() {
executorRunning.incrementAndGet();
try {
- Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent, acceptLanguage, pdfPath, wkhtmltopdfTimeout);
+ Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent(), acceptLanguage, pdfPath, wkhtmltopdfTimeout);
} catch (Throwable e) {} finally {
executorRunning.decrementAndGet();
}
@@ -245,9 +245,9 @@ public class Transactions {
};
executor.execute(t);
} else {
- success = Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent, acceptLanguage, pdfPath, wkhtmltopdfTimeout);
+ success = Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent(), acceptLanguage, pdfPath, wkhtmltopdfTimeout);
}
-
+
return success;
}
@@ -270,7 +270,7 @@ public class Transactions {
public static Revisions rollback(String urlhash) {
return transact(urlhash, State.ARCHIVE, State.INVENTORY);
}
-
+
private static Revisions transact(final String urlhash, final State from, final State to) {
Revisions r = Transactions.getRevisions(from, urlhash);
if (r == null) return null;
@@ -303,10 +303,10 @@ public class Transactions {
} catch (MalformedURLException e) {
ConcurrentLog.logException(e);
}
-
+
return null;
}
-
+
/**
* select a set of urlhashes from the snapshot directory. The selection either ordered
* by generation date (upwards == OLDESTFIRST or downwards == LATESTFIRST) or with any
@@ -315,7 +315,7 @@ public class Transactions {
* @param depth selected depth or null for all depths
* @param order Order.ANY, Order.OLDESTFIRST or Order.LATESTFIRST
* @param maxcount the maximum number of hosthashes. If unlimited, submit Integer.MAX_VALUE
- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
* @return a map of hosthashes with the associated creation date
*/
public static LinkedHashMap<String, Revisions> select(String host, Integer depth, final Order order, int maxcount, State state) {
@@ -332,7 +332,7 @@ public class Transactions {
* @param depth
* @param date
* @param ext
- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
* @return a file to the snapshot
*/
public static File definePath(final DigestURL url, final int depth, final Date date, final String ext, State state) {
@@ -345,25 +345,25 @@ public class Transactions {
/**
* Write information about the storage of a snapshot to the Snapshot-internal index.
* The actual writing of files to the target directory must be done elsewehre, this method does not store the snapshot files.
- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
* @param url
* @param depth
* @param date
*/
public static void announceStorage(final DigestURL url, final int depth, final Date date, State state) {
- if (state == State.INVENTORY || state == State.ANY) inventory.announceStorage(url, depth, date);
+ if (state == State.INVENTORY || state == State.ANY) inventory.announceStorage(url, depth, date);
if (state == State.ARCHIVE || state == State.ANY) archive.announceStorage(url, depth, date);
}
/**
* Delete information about the storage of a snapshot to the Snapshot-internal index.
* The actual deletion of files in the target directory must be done elsewehre, this method does not store the snapshot files.
- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
* @param url
* @param depth
*/
public static void announceDeletion(final DigestURL url, final int depth, final State state) {
- if (state == State.INVENTORY || state == State.ANY) inventory.announceDeletion(url, depth);
+ if (state == State.INVENTORY || state == State.ANY) inventory.announceDeletion(url, depth);
if (state == State.ARCHIVE || state == State.ANY) archive.announceDeletion(url, depth);
}
@@ -374,7 +374,7 @@ public class Transactions {
* findPaths/3 with a given depth.
* @param url
* @param ext required extension or null if the extension must not be checked
- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
* @return a set of files for snapshots of the url
*/
public static Collection<File> findPaths(final DigestURL url, final String ext, State state) {
@@ -383,14 +383,14 @@ public class Transactions {
if (state == State.ARCHIVE || state == State.ANY) result.addAll(archive.findPaths(url, ext));
return result;
}
-
+
/**
* for a given url, get all paths for storage locations.
* The locations are all for the single url but may represent different storage times.
* @param url
* @param ext required extension or null if the extension must not be checked
* @param depth
- * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
+ * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
* @return a set of files for snapshots of the url
*/
public static Collection<File> findPaths(final DigestURL url, final int depth, final String ext, State state) {
@@ -399,5 +399,5 @@ public class Transactions {
if (state == State.ARCHIVE || state == State.ANY) result.addAll(archive.findPaths(url, depth, ext));
return result;
}
-
+
}
diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.java
index c5e7ef09c..af963473c 100644
--- a/source/net/yacy/crawler/retrieval/HTTPLoader.java
+++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java
@@ -308,7 +308,7 @@ public final class HTTPLoader {
private RequestHeader createRequestheader(final Request request, final ClientIdentification.Agent agent)
throws IOException {
final RequestHeader requestHeader = new RequestHeader();
- requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent);
+ requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent());
if (request.referrerhash() != null) {
String refererURL = this.sb.getURL(request.referrerhash());
if (refererURL != null) {
@@ -478,7 +478,7 @@ public final class HTTPLoader {
// create a request header
final RequestHeader requestHeader = new RequestHeader();
- requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent);
+ requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent());
requestHeader.put(HeaderFramework.ACCEPT_LANGUAGE, DEFAULT_LANGUAGE);
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, DEFAULT_ENCODING);
diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.java
index 2035e961c..0bc5a3928 100644
--- a/source/net/yacy/crawler/robots/RobotsTxt.java
+++ b/source/net/yacy/crawler/robots/RobotsTxt.java
@@ -87,7 +87,7 @@ public class RobotsTxt {
public RobotsTxt(final WorkTables worktables, LoaderDispatcher loader, final int maxActiveTheads) {
this.threadPool = new ThreadPoolExecutor(maxActiveTheads, maxActiveTheads,
0L, TimeUnit.MILLISECONDS,
- new LinkedBlockingQueue<Runnable>(),
+ new LinkedBlockingQueue<>(),
new NamePrefixThreadFactory(RobotsTxt.class.getSimpleName()));
this.syncObjects = new ConcurrentHashMap<>();
this.tables = worktables;
@@ -199,7 +199,7 @@ public class RobotsTxt {
if (response == null) {
this.processOldEntry(robotsTxt4Host, robotsURL, robotsTable);
} else {
- robotsTxt4Host = this.processNewEntry(robotsURL, response, agent.robotIDs);
+ robotsTxt4Host = this.processNewEntry(robotsURL, response, agent.robotIDs());
}
}
}
@@ -225,6 +225,7 @@ public class RobotsTxt {
}
public void ensureExist(final MultiProtocolURL theURL, final ClientIdentification.Agent agent, boolean concurrent) {
+ if (!agent.isRobot()) return;
if (theURL.isLocal()) return;
final String urlHostPort = getHostPort(theURL);
if (urlHostPort == null) return;
@@ -266,7 +267,7 @@ public class RobotsTxt {
if (response == null) {
RobotsTxt.this.processOldEntry(null, robotsURL, robotsTable);
} else {
- RobotsTxt.this.processNewEntry(robotsURL, response, agent.robotIDs);
+ RobotsTxt.this.processNewEntry(robotsURL, response, agent.robotIDs());
}
}
}
@@ -292,8 +293,8 @@ public class RobotsTxt {
// generate artificial entry
robotsTxt4Host = new RobotsTxtEntry(
robotsURL,
- new ArrayList<String>(),
- new ArrayList<String>(),
+ new ArrayList<>(),
+ new ArrayList<>(),
new Date(),
new Date(),
null,
diff --git a/source/net/yacy/htroot/CrawlCheck_p.java b/source/net/yacy/htroot/CrawlCheck_p.java
index d32c6f442..c483d36f3 100644
--- a/source/net/yacy/htroot/CrawlCheck_p.java
+++ b/source/net/yacy/htroot/CrawlCheck_p.java
@@ -47,7 +47,7 @@ public class CrawlCheck_p {
if (post.containsKey("crawlcheck")) {
// get the list of rootURls for this crawl start
- final Set<DigestURL> rootURLs = new LinkedHashSet<DigestURL>();
+ final Set<DigestURL> rootURLs = new LinkedHashSet<>();
final String crawlingStart0 = post.get("crawlingURLs","").trim();
final String[] rootURLs0 = crawlingStart0.indexOf('\n') > 0 || crawlingStart0.indexOf('\r') > 0 ? crawlingStart0.split("[\\r\\n]+") : crawlingStart0.split(Pattern.quote("|"));
for (String crawlingStart: rootURLs0) {
@@ -89,12 +89,12 @@ public class CrawlCheck_p {
boolean robotsAllowed = true;
if (entry.robotsTxtEntry == null) {
prop.put("table_list_" + row + "_robots", "no robots");
- prop.put("table_list_" + row + "_crawldelay", agent.minimumDelta + " ms");
+ prop.put("table_list_" + row + "_crawldelay", agent.minimumDelta() + " ms");
prop.put("table_list_" + row + "_sitemap", "");
} else {
robotsAllowed = !entry.robotsTxtEntry.isDisallowed(entry.digestURL);
prop.put("table_list_" + row + "_robots", "robots exist: " + (robotsAllowed ? "crawl allowed" : "url disallowed"));
- prop.put("table_list_" + row + "_crawldelay", Math.max(agent.minimumDelta, entry.robotsTxtEntry.getCrawlDelayMillis()) + " ms");
+ prop.put("table_list_" + row + "_crawldelay", Math.max(agent.minimumDelta(), entry.robotsTxtEntry.getCrawlDelayMillis()) + " ms");
prop.put("table_list_" + row + "_sitemap", entry.robotsTxtEntry.getSitemaps().toString());
}
diff --git a/source/net/yacy/htroot/CrawlStartExpert.java b/source/net/yacy/htroot/CrawlStartExpert.java
index b382d3300..c0d0f3a38 100644
--- a/source/net/yacy/htroot/CrawlStartExpert.java
+++ b/source/net/yacy/htroot/CrawlStartExpert.java
@@ -572,17 +572,15 @@ public class CrawlStartExpert {
// ---------- Agent name
final List<String> agentNames = new ArrayList<>();
+ agentNames.add(ClientIdentification.browserAgentName); // see https://github.com/yacy/yacy_search_server/issues/727
+ agentNames.add(ClientIdentification.googleAgentName); // see https://github.com/yacy/yacy_search_server/issues/727
if (sb.isIntranetMode()) {
agentNames.add(ClientIdentification.yacyIntranetCrawlerAgentName);
}
if (sb.isGlobalMode()) {
agentNames.add(ClientIdentification.yacyInternetCrawlerAgentName);
}
- agentNames.add(ClientIdentification.googleAgentName);
- if (sb.isAllIPMode()) {
- agentNames.add(ClientIdentification.browserAgentName);
- if (ClientIdentification.getAgent(ClientIdentification.customAgentName) != null) agentNames.add(ClientIdentification.customAgentName);
- }
+ if (ClientIdentification.getAgent(ClientIdentification.customAgentName) != null) agentNames.add(ClientIdentification.customAgentName);
String defaultAgentName = agentNames.get(0);
if (post != null && post.containsKey("agentName")) {
final String agentName = post.get("agentName", sb.isIntranetMode() ? ClientIdentification.yacyIntranetCrawlerAgentName : ClientIdentification.yacyInternetCrawlerAgentName);
diff --git a/source/net/yacy/htroot/Network.java b/source/net/yacy/htroot/Network.java
index 64519f686..b8cc05390 100644
--- a/source/net/yacy/htroot/Network.java
+++ b/source/net/yacy/htroot/Network.java
@@ -222,7 +222,7 @@ public class Network {
return prop;
}
- final ConcurrentMap<String, String> map = new ConcurrentHashMap<String, String>();
+ final ConcurrentMap<String, String> map = new ConcurrentHashMap<>();
final String challengeIP = post.get("peerIP");
final String challengePort = post.get("peerPort");
map.put(Seed.IP, challengeIP);
@@ -302,10 +302,10 @@ public class Network {
}
// find updated Information using YaCyNews
- final HashSet<String> updatedProfile = new HashSet<String>();
- final HashMap<String, Map<String, String>> updatedWiki = new HashMap<String, Map<String, String>>();
- final HashMap<String, Map<String, String>> updatedBlog = new HashMap<String, Map<String, String>>();
- final HashMap<String, String> isCrawling = new HashMap<String, String>();
+ final HashSet<String> updatedProfile = new HashSet<>();
+ final HashMap<String, Map<String, String>> updatedWiki = new HashMap<>();
+ final HashMap<String, Map<String, String>> updatedBlog = new HashMap<>();
+ final HashMap<String, String> isCrawling = new HashMap<>();
NewsDB.Record record;
final Iterator<NewsDB.Record> recordIterator = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);
while (recordIterator.hasNext()) {
@@ -430,7 +430,7 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_ssl_portssl", seed.get(Seed.PORTSSL,"8443"));
userAgent = null;
if (seed.hash != null && seed.hash.equals(sb.peers.mySeed().hash)) {
- userAgent = ClientIdentification.yacyInternetCrawlerAgent.userAgent;
+ userAgent = ClientIdentification.yacyInternetCrawlerAgent.userAgent();
location = ClientIdentification.generateLocation();
} else {
userAgent = sb.peers.peerActions.getUserAgent(ip);
diff --git a/source/net/yacy/htroot/api/latency_p.java b/source/net/yacy/htroot/api/latency_p.java
index c799a32d5..aedf5e048 100644
--- a/source/net/yacy/htroot/api/latency_p.java
+++ b/source/net/yacy/htroot/api/latency_p.java
@@ -55,7 +55,7 @@ public class latency_p {
prop.put("domains_" + c + "_count", host.count());
prop.put("domains_" + c + "_average", host.average());
prop.put("domains_" + c + "_robots", host.robotsDelay());
- prop.put("domains_" + c + "_flux", host.flux(agent.minimumDelta));
+ prop.put("domains_" + c + "_flux", host.flux(agent.minimumDelta()));
c++;
}
prop.put("domains", c);
diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java
index ca0cd0a35..38c82d210 100644
--- a/source/net/yacy/peers/SeedDB.java
+++ b/source/net/yacy/peers/SeedDB.java
@@ -122,9 +122,9 @@ public final class SeedDB implements AlternativeDomainNames {
this.seedPotentialDB = openSeedTable(this.seedPotentialDBFile);
// check if we are in the seedCaches: this can happen if someone else published our seed
- removeMySeed();
+ this.removeMySeed();
- this.lastSeedUpload_seedDBSize = sizeConnected();
+ this.lastSeedUpload_seedDBSize = this.sizeConnected();
// create or init news database
this.newsPool = new NewsPool(networkRoot, useTailCache, exceed134217727);
@@ -154,7 +154,7 @@ public final class SeedDB implements AlternativeDomainNames {
// replace my (old) seed with new seed definition from other network
// but keep the seed name
- final String peername = myName();
+ final String peername = this.myName();
this.mySeed = null; // my own seed
this.myOwnSeedFile = new File(newNetworkRoot, SeedDB.DBFILE_OWN_SEED);
@@ -166,13 +166,13 @@ public final class SeedDB implements AlternativeDomainNames {
this.seedPassiveDB = openSeedTable(this.seedPassiveDBFile);
this.seedPotentialDB = openSeedTable(this.seedPotentialDBFile);
- initMySeed();
+ this.initMySeed();
this.mySeed.setName(peername);
// check if we are in the seedCaches: this can happen if someone else published our seed
- removeMySeed();
+ this.removeMySeed();
- this.lastSeedUpload_seedDBSize = sizeConnected();
+ this.lastSeedUpload_seedDBSize = this.sizeConnected();
// create or init news database
this.newsPool = new NewsPool(newNetworkRoot, useTailCache, exceed134217727);
@@ -227,49 +227,49 @@ public final class SeedDB implements AlternativeDomainNames {
public Seed mySeed() {
if (this.mySeed == null) {
- if (sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init
- initMySeed();
+ if (this.sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init
+ this.initMySeed();
}
return this.mySeed;
}
public void setMyName(final String name) {
- mySeed().setName(name);
+ this.mySeed().setName(name);
}
@Override
public String myAlternativeAddress() {
- return mySeed().getName() + ".yacy";
+ return this.mySeed().getName() + ".yacy";
}
@Deprecated
public String myIP() {
- return mySeed().getIP();
+ return this.mySeed().getIP();
}
-
+
@Override
public Set<String> myIPs() {
- return mySeed().getIPs();
+ return this.mySeed().getIPs();
}
@Override
public int myPort() {
- return mySeed().getPort();
+ return this.mySeed().getPort();
}
@Override
public String myName() {
- return mySeed().getName();
+ return this.mySeed().getName();
}
@Override
public String myID() {
- return mySeed().hash;
+ return this.mySeed().hash;
}
public synchronized void removeMySeed() {
if (this.seedActiveDB.isEmpty() && this.seedPassiveDB.isEmpty() && this.seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early
- if (this.mySeed == null) initMySeed();
+ if (this.mySeed == null) this.initMySeed();
try {
final byte[] mySeedHash = ASCII.getBytes(this.mySeed.hash);
this.seedActiveDB.delete(mySeedHash);
@@ -280,13 +280,13 @@ public final class SeedDB implements AlternativeDomainNames {
public void saveMySeed() {
try {
- mySeed().save(this.myOwnSeedFile);
+ this.mySeed().save(this.myOwnSeedFile);
} catch (final IOException e) { ConcurrentLog.warn("yacySeedDB", "could not save mySeed '"+ this.myOwnSeedFile +"': "+ e.getMessage()); }
}
public boolean noDHTActivity() {
// for small networks, we don't perform DHT transmissions, because it is possible to search over all peers
- return sizeConnected() <= dhtActivityMagic;
+ return this.sizeConnected() <= dhtActivityMagic;
}
private synchronized static MapDataMining openSeedTable(final File seedDBFile) {
@@ -323,9 +323,9 @@ public final class SeedDB implements AlternativeDomainNames {
return seedDB;
}
- public synchronized void resetActiveTable() { this.seedActiveDB = resetSeedTable(this.seedActiveDB, this.seedActiveDBFile); }
- private synchronized void resetPassiveTable() { this.seedPassiveDB = resetSeedTable(this.seedPassiveDB, this.seedPassiveDBFile); }
- private synchronized void resetPotentialTable() { this.seedPotentialDB = resetSeedTable(this.seedPotentialDB, this.seedPotentialDBFile); }
+ public synchronized void resetActiveTable() { this.seedActiveDB = this.resetSeedTable(this.seedActiveDB, this.seedActiveDBFile); }
+ private synchronized void resetPassiveTable() { this.seedPassiveDB = this.resetSeedTable(this.seedPassiveDB, this.seedPassiveDBFile); }
+ private synchronized void resetPotentialTable() { this.seedPotentialDB = this.resetSeedTable(this.seedPotentialDB, this.seedPotentialDBFile); }
public void close() {
if (this.seedActiveDB != null) this.seedActiveDB.close();
@@ -374,7 +374,7 @@ public final class SeedDB implements AlternativeDomainNames {
if (yacydom.endsWith(".yacyh")) {
// find a peer with its hexhash
hash = Seed.hexHash2b64Hash(yacydom.substring(0, yacydom.length() - 6));
- seed = get(hash);
+ seed = this.get(hash);
if (seed == null) {
Network.log.warn("cluster peer '" + yacydom + "' was not found.");
} else {
@@ -382,7 +382,7 @@ public final class SeedDB implements AlternativeDomainNames {
}
} else if (yacydom.endsWith(".yacy")) {
// find a peer with its name
- seed = lookupByName(yacydom.substring(0, yacydom.length() - 5));
+ seed = this.lookupByName(yacydom.substring(0, yacydom.length() - 5));
if (seed == null) {
Network.log.warn("cluster peer '" + yacydom + "' was not found.");
} else {
@@ -407,14 +407,14 @@ public final class SeedDB implements AlternativeDomainNames {
public Seed anySeedVersion(final double minVersion) {
// return just any seed that has a specific minimum version number
- final Iterator<Seed> e = seedsConnected(true, true, Seed.randomHash(), minVersion);
+ final Iterator<Seed> e = this.seedsConnected(true, true, Seed.randomHash(), minVersion);
return e.next();
}
private final Map<Long, Integer> sizeActiveSince_result = new ConcurrentHashMap<>();
private final Map<Long, Long> sizeActiveSince_time = new ConcurrentHashMap<>();
-
+
/**
* count the number of peers that had been seen within the given time limit
* @param limitMinutes the time limit in minutes. 1440 minutes is a day
@@ -422,22 +422,22 @@ public final class SeedDB implements AlternativeDomainNames {
*/
public int sizeActiveSince(final long limitMinutes) {
// buffering because that iteration takes a long time
- long now = System.currentTimeMillis();
- Integer s0 = sizeActiveSince_result.get(limitMinutes);
- Long t0 = sizeActiveSince_time.get(limitMinutes);
+ final long now = System.currentTimeMillis();
+ final Integer s0 = this.sizeActiveSince_result.get(limitMinutes);
+ final Long t0 = this.sizeActiveSince_time.get(limitMinutes);
if (s0 == null || t0 == null || now - t0 > 60000) {
- int s1 = sizeActiveSinceInt(limitMinutes);
- sizeActiveSince_result.put(limitMinutes, s1);
- sizeActiveSince_time.put(limitMinutes, now);
+ final int s1 = this.sizeActiveSinceInt(limitMinutes);
+ this.sizeActiveSince_result.put(limitMinutes, s1);
+ this.sizeActiveSince_time.put(limitMinutes, now);
return s1;
}
- return s0;
+ return s0;
}
-
+
private int sizeActiveSinceInt(final long limitMinutes) {
int c = this.seedActiveDB.size();
Seed seed;
- Iterator<Seed> i = seedsSortedDisconnected(false, Seed.LASTSEEN);
+ Iterator<Seed> i = this.seedsSortedDisconnected(false, Seed.LASTSEEN);
while (i.hasNext()) {
seed = i.next();
if (seed != null) {
@@ -445,7 +445,7 @@ public final class SeedDB implements AlternativeDomainNames {
c++;
}
}
- i = seedsSortedPotential(false, Seed.LASTSEEN);
+ i = this.seedsSortedPotential(false, Seed.LASTSEEN);
while (i.hasNext()) {
seed = i.next();
if (seed != null) {
@@ -476,7 +476,7 @@ public final class SeedDB implements AlternativeDomainNames {
public long countPassiveRWI() { return this.seedPassiveDB.getLongAcc(Seed.ICOUNT); }
public long countPotentialURL() { return this.seedPotentialDB.getLongAcc(Seed.LCOUNT); }
public long countPotentialRWI() { return this.seedPotentialDB.getLongAcc(Seed.ICOUNT); }
-
+
public void addConnected(final Seed seed) {
if (seed.isProper(false) != null) return;
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
@@ -488,7 +488,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.seedPotentialDB.delete(ASCII.getBytes(seed.hash));
} catch (final Exception e) {
Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- resetActiveTable();
+ this.resetActiveTable();
}
}
}
@@ -506,7 +506,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.seedPassiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
} catch (final Exception e) {
Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- resetPassiveTable();
+ this.resetPassiveTable();
}
}
}
@@ -524,7 +524,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.seedPotentialDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
} catch (final Exception e) {
Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- resetPotentialTable();
+ this.resetPotentialTable();
}
}
}
@@ -558,7 +558,7 @@ public final class SeedDB implements AlternativeDomainNames {
private Seed get(final String hash, final MapDataMining database) {
if (hash == null || hash.isEmpty()) return null;
if ((this.mySeed != null) && (hash.equals(this.mySeed.hash))) return this.mySeed;
- final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();
+ final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<>();
try {
final Map<String, String> map = database.get(ASCII.getBytes(hash));
if (map == null) return null;
@@ -584,7 +584,7 @@ public final class SeedDB implements AlternativeDomainNames {
private Seed get(final byte[] hash, final MapDataMining database) {
if (hash == null || hash.length == 0) return null;
if ((this.mySeed != null) && (ASCII.String(hash).equals(this.mySeed.hash))) return this.mySeed;
- final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();
+ final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<>();
try {
final Map<String, String> map = database.get(hash);
if (map == null) return null;
@@ -600,33 +600,33 @@ public final class SeedDB implements AlternativeDomainNames {
}
public Seed getConnected(final String hash) {
- return get(hash, this.seedActiveDB);
+ return this.get(hash, this.seedActiveDB);
}
public Seed getConnected(final byte[] hash) {
- return get(hash, this.seedActiveDB);
+ return this.get(hash, this.seedActiveDB);
}
public Seed getDisconnected(final String hash) {
- return get(hash, this.seedPassiveDB);
+ return this.get(hash, this.seedPassiveDB);
}
public Seed getDisconnected(final byte[] hash) {
- return get(hash, this.seedPassiveDB);
+ return this.get(hash, this.seedPassiveDB);
}
public Seed getPotential(final String hash) {
- return get(hash, this.seedPotentialDB);
+ return this.get(hash, this.seedPotentialDB);
}
public Seed getPotential(final byte[] hash) {
- return get(hash, this.seedPotentialDB);
+ return this.get(hash, this.seedPotentialDB);
}
public Seed get(final String hash) {
- Seed seed = getConnected(hash);
- if (seed == null) seed = getDisconnected(hash);
- if (seed == null) seed = getPotential(hash);
+ Seed seed = this.getConnected(hash);
+ if (seed == null) seed = this.getDisconnected(hash);
+ if (seed == null) seed = this.getPotential(hash);
return seed;
}
@@ -638,7 +638,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.seedActiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
} catch (final Exception e) {
Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- resetActiveTable();
+ this.resetActiveTable();
}
}
}
@@ -650,7 +650,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.seedPassiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
} catch (final Exception e) {
Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- resetActiveTable();
+ this.resetActiveTable();
}
}
}
@@ -662,7 +662,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.seedPotentialDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
} catch (final Exception e) {
Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- resetActiveTable();
+ this.resetActiveTable();
}
}
}
@@ -673,7 +673,7 @@ public final class SeedDB implements AlternativeDomainNames {
// local peer?
if (peerName.equals("localpeer")) {
- if (this.mySeed == null) initMySeed();
+ if (this.mySeed == null) this.initMySeed();
return this.mySeed;
}
@@ -683,8 +683,8 @@ public final class SeedDB implements AlternativeDomainNames {
// enumerate the cache
String name = Seed.checkPeerName(peerName);
synchronized (this) { try {
- Collection<byte[]> idx = this.seedActiveDB.select(Seed.NAME, name);
- for (byte[] pk: idx) {
+ final Collection<byte[]> idx = this.seedActiveDB.select(Seed.NAME, name);
+ for (final byte[] pk: idx) {
seed = this.getConnected(pk);
if (seed == null) continue;
//System.out.println("*** found lookupByName in seedActiveDB: " + peerName);
@@ -693,8 +693,8 @@ public final class SeedDB implements AlternativeDomainNames {
} catch (final IOException e ) {
}}
synchronized (this) { try {
- Collection<byte[]> idx = this.seedPassiveDB.select(Seed.NAME, name);
- for (byte[] pk: idx) {
+ final Collection<byte[]> idx = this.seedPassiveDB.select(Seed.NAME, name);
+ for (final byte[] pk: idx) {
seed = this.getDisconnected(pk);
if (seed == null) continue;
//System.out.println("*** found lookupByName in seedPassiveDB: " + peerName);
@@ -704,7 +704,7 @@ public final class SeedDB implements AlternativeDomainNames {
}}
// check local seed
- if (this.mySeed == null) initMySeed();
+ if (this.mySeed == null) this.initMySeed();
name = this.mySeed.getName().toLowerCase();
if (name.equals(peerName)) return this.mySeed;
// nothing found
@@ -718,13 +718,13 @@ public final class SeedDB implements AlternativeDomainNames {
final boolean lookupDisconnected,
final boolean lookupPotential
) {
- for (String i: peerIPs) {
- Seed s = lookupByIP(Domains.dnsResolve(i), port, lookupConnected, lookupDisconnected, lookupPotential);
+ for (final String i: peerIPs) {
+ final Seed s = this.lookupByIP(Domains.dnsResolve(i), port, lookupConnected, lookupDisconnected, lookupPotential);
if (s != null) return s;
}
return null;
}
-
+
public Seed lookupByIP(
final InetAddress peerIP,
final int port, /* port may be -1 if not significant */
@@ -737,18 +737,18 @@ public final class SeedDB implements AlternativeDomainNames {
// local peer?
if (Domains.isThisHostIP(peerIP)) {
- if (this.mySeed == null) initMySeed();
+ if (this.mySeed == null) this.initMySeed();
if (port > 0 && this.mySeed.getPort() == port) return this.mySeed;
}
// then try to use the cache
Seed seed = null;
- String ipString = peerIP.getHostAddress();
+ final String ipString = peerIP.getHostAddress();
if (lookupConnected) {
try {
- Collection<byte[]> idx = this.seedActiveDB.select(Seed.IP, ipString);
- for (byte[] pk: idx) {
+ final Collection<byte[]> idx = this.seedActiveDB.select(Seed.IP, ipString);
+ for (final byte[] pk: idx) {
seed = this.getConnected(pk);
if (seed == null) continue;
if (port > 0 && seed.getPort() != port) continue;
@@ -762,8 +762,8 @@ public final class SeedDB implements AlternativeDomainNames {
if (lookupDisconnected) {
try {
- Collection<byte[]> idx = this.seedPassiveDB.select(Seed.IP, ipString);
- for (byte[] pk: idx) {
+ final Collection<byte[]> idx = this.seedPassiveDB.select(Seed.IP, ipString);
+ for (final byte[] pk: idx) {
seed = this.getDisconnected(pk);
if (seed == null) continue;
if (port > 0 && seed.getPort() != port) continue;
@@ -777,8 +777,8 @@ public final class SeedDB implements AlternativeDomainNames {
if (lookupPotential) {
try {
- Collection<byte[]> idx = this.seedPotentialDB.select(Seed.IP, ipString);
- for (byte[] pk: idx) {
+ final Collection<byte[]> idx = this.seedPotentialDB.select(Seed.IP, ipString);
+ for (final byte[] pk: idx) {
seed = this.getPotential(pk);
if (seed == null) continue;
if (port > 0 && seed.getPort() != port) continue;
@@ -792,7 +792,7 @@ public final class SeedDB implements AlternativeDomainNames {
// check local seed
if (this.mySeed == null || !this.mySeed.getIPs().contains(ipString)) return null;
- int p = this.mySeed.getPort();
+ final int p = this.mySeed.getPort();
if (port > 0 && p != port) return null;
//System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());
return this.mySeed;
@@ -800,13 +800,13 @@ public final class SeedDB implements AlternativeDomainNames {
private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException {
PrintWriter pw = null;
- final ArrayList<String> v = new ArrayList<String>(this.seedActiveDB.size() + 1);
+ final ArrayList<String> v = new ArrayList<>(this.seedActiveDB.size() + 1);
try {
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
- List<Seed> seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed, false, 0.0f);
+ final List<Seed> seedlist = this.getSeedlist(Integer.MAX_VALUE, addMySeed, false, 0.0f);
String line;
- for (Seed seed: seedlist) {
+ for (final Seed seed: seedlist) {
line = seed.genSeedStr(null);
v.add(line);
pw.print(line + serverCore.CRLF_STRING);
@@ -819,8 +819,8 @@ public final class SeedDB implements AlternativeDomainNames {
}
public ArrayList<Seed> getSeedlist(int maxcount, boolean addMySeed, boolean nodeonly, float minversion) {
- final ArrayList<Seed> v = new ArrayList<Seed>(this.seedActiveDB.size() + 1000);
-
+ final ArrayList<Seed> v = new ArrayList<>(this.seedActiveDB.size() + 1000);
+
// store own peer seed
if (addMySeed) v.add(this.mySeed);
@@ -841,12 +841,12 @@ public final class SeedDB implements AlternativeDomainNames {
}
final StringBuilder encoded = new StringBuilder(1024);
- for (Seed seed: v) {
+ for (final Seed seed: v) {
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
}
return v;
}
-
+
protected String uploadSeedList(final yacySeedUploader uploader,
final serverSwitch sb,
final SeedDB seedDB,
@@ -861,7 +861,7 @@ public final class SeedDB implements AlternativeDomainNames {
// create a seed file which for uploading ...
seedFile = File.createTempFile("seedFile",".txt", seedDB.myOwnSeedFile.getParentFile());
if (Network.log.isFine()) Network.log.fine("SaveSeedList: Storing seedlist into tempfile " + seedFile.toString());
- final ArrayList<String> uv = storeSeedList(seedFile, true);
+ final ArrayList<String> uv = this.storeSeedList(seedFile, true);
// uploading the seed file
if (Network.log.isFine()) Network.log.fine("SaveSeedList: Trying to upload seed-file, " + seedFile.length() + " bytes, " + uv.size() + " entries.");
@@ -895,7 +895,7 @@ public final class SeedDB implements AlternativeDomainNames {
final RequestHeader reqHeader = new RequestHeader();
reqHeader.put(HeaderFramework.PRAGMA, "no-cache");
reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache, no-store"); // httpc uses HTTP/1.0 is this necessary?
- reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);
+ reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent());
byte[] content = null;
try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
@@ -906,7 +906,7 @@ public final class SeedDB implements AlternativeDomainNames {
} catch (final Exception e) {
throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
}
-
+
// check response code
if (client.getHttpResponse().getStatusLine().getStatusCode() != 200) {
throw new IOException("Server returned status: " + client.getHttpResponse().getStatusLine());
@@ -967,10 +967,10 @@ public final class SeedDB implements AlternativeDomainNames {
hash = Seed.hexHash2b64Hash(hash);
}
// check remote seeds
- seed = getConnected(hash); // checks only remote, not local
+ seed = this.getConnected(hash); // checks only remote, not local
// check local seed
if (seed == null) {
- if (this.mySeed == null) initMySeed();
+ if (this.mySeed == null) this.initMySeed();
if (hash.equals(this.mySeed.hash))
seed = this.mySeed;
else return null;
@@ -989,9 +989,9 @@ public final class SeedDB implements AlternativeDomainNames {
}
// identify domain
final String domain = host.substring(0, host.length() - 5).toLowerCase(Locale.ROOT);
- seed = lookupByName(domain);
+ seed = this.lookupByName(domain);
if (seed == null) return null;
- if (this.mySeed == null) initMySeed();
+ if (this.mySeed == null) this.initMySeed();
if (seed == this.mySeed && !(seed.isOnline())) {
// take local ip instead of external
final Set<String> ips = Switchboard.getSwitchboard().myPublicIPs();
@@ -1024,7 +1024,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.it = (firstKey == null) ? database.entries(up, rot) : database.entries(up, rot, firstKey, secondKey);
double version;
while (true) {
- this.nextSeed = internalNext();
+ this.nextSeed = this.internalNext();
if (this.nextSeed == null) break;
version = this.nextSeed.getVersion();
if (version >= this.minVersion || version == 0.0) break; // include 0.0 to access always developer peers
@@ -1032,14 +1032,14 @@ public final class SeedDB implements AlternativeDomainNames {
} catch (final IOException e) {
ConcurrentLog.logException(e);
Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
- if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
+ if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
+ if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
this.it = null;
} catch (final kelondroException e) {
ConcurrentLog.logException(e);
Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
- if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
+ if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
+ if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
this.it = null;
}
}
@@ -1048,13 +1048,13 @@ public final class SeedDB implements AlternativeDomainNames {
this.database = database;
try {
this.it = database.entries(up, field);
- this.nextSeed = internalNext();
+ this.nextSeed = this.internalNext();
} catch (final kelondroException e) {
ConcurrentLog.logException(e);
Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
- if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
- if (database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);
+ if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
+ if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
+ if (database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);
this.it = null;
}
}
@@ -1084,9 +1084,9 @@ public final class SeedDB implements AlternativeDomainNames {
} catch (final Exception e) {
ConcurrentLog.logException(e);
Network.log.severe("ERROR internalNext: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
- if (this.database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
- if (this.database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
- if (this.database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);
+ if (this.database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
+ if (this.database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
+ if (this.database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = SeedDB.this.resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);
return null;
}
}
@@ -1096,7 +1096,7 @@ public final class SeedDB implements AlternativeDomainNames {
final Seed seed = this.nextSeed;
double version;
try {while (true) {
- this.nextSeed = internalNext();
+ this.nextSeed = this.internalNext();
if (this.nextSeed == null) break;
version = this.nextSeed.getVersion();
if (version >= this.minVersion || version == 0.0) break; // include 0.0 to access always developer peers
@@ -1121,12 +1121,12 @@ public final class SeedDB implements AlternativeDomainNames {
public void loadSeedListConcurrently(final String seedListFileURL, final AtomicInteger scc, final int timeout, final boolean checkAge) {
// uses the superseed to initialize the database with known seeds
- Thread seedLoader = new Thread("SeedDB.loadSeedListConcurrently") {
+ final Thread seedLoader = new Thread("SeedDB.loadSeedListConcurrently") {
@Override
public void run() {
// load the seed list
try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout)) {
- DigestURL url = new DigestURL(seedListFileURL);
+ final DigestURL url = new DigestURL(seedListFileURL);
//final long start = System.currentTimeMillis();
final RequestHeader reqHeader = new RequestHeader();
reqHeader.put(HeaderFramework.PRAGMA, "no-cache");
@@ -1134,8 +1134,8 @@ public final class SeedDB implements AlternativeDomainNames {
client.setHeader(reqHeader.entrySet());
client.HEADResponse(url.toNormalform(false), false);
- int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
- ResponseHeader header = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
+ final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
+ final ResponseHeader header = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
if (checkAge) {
if (header.lastModified() == null) {
Network.log.warn("BOOTSTRAP: seed-list URL " + seedListFileURL + " not usable, last-modified is missing");
@@ -1147,17 +1147,17 @@ public final class SeedDB implements AlternativeDomainNames {
}
scc.incrementAndGet();
final byte[] content = client.GETbytes(url, null, null, false);
- Iterator<String> enu = FileUtils.strings(content);
+ final Iterator<String> enu = FileUtils.strings(content);
int lc = 0;
int uptodatec = 0, outdatedc = 0;
while (enu.hasNext()) {
try {
- Seed ys = Seed.genRemoteSeed(enu.next(), false, null);
- if ((ys != null) && (!mySeedIsDefined() || !mySeed().hash.equals(ys.hash))) {
+ final Seed ys = Seed.genRemoteSeed(enu.next(), false, null);
+ if ((ys != null) && (!SeedDB.this.mySeedIsDefined() || !SeedDB.this.mySeed().hash.equals(ys.hash))) {
final long lastseen = Math.abs((System.currentTimeMillis() - ys.getLastSeenUTC()) / 1000 / 60);
if (lastseen < 240) uptodatec++; else outdatedc++;
if ( lastseen < 240 || lc < 10 ) {
- if (peerActions.connectPeer(ys, false) ) lc++;
+ if (SeedDB.this.peerActions.connectPeer(ys, false) ) lc++;
}
}
} catch (final Throwable e ) {
@@ -1177,5 +1177,5 @@ public final class SeedDB implements AlternativeDomainNames {
};
seedLoader.start();
}
-
+
}
diff --git a/source/net/yacy/repository/LoaderDispatcher.java b/source/net/yacy/repository/LoaderDispatcher.java
index 1d1439482..2b3ce1c0d 100644
--- a/source/net/yacy/repository/LoaderDispatcher.java
+++ b/source/net/yacy/repository/LoaderDispatcher.java
@@ -71,7 +71,7 @@ public final class LoaderDispatcher {
private final static int accessTimeMaxsize = 1000;
private final static ConcurrentLog log = new ConcurrentLog("LOADER");
- private static final ConcurrentHashMap<String, Long> accessTime = new ConcurrentHashMap<String, Long>(); // to protect targets from DDoS
+ private static final ConcurrentHashMap<String, Long> accessTime = new ConcurrentHashMap<>(); // to protect targets from DDoS
private final Switchboard sb;
private final HashSet<String> supportedProtocols;
@@ -83,14 +83,14 @@ public final class LoaderDispatcher {
public LoaderDispatcher(final Switchboard sb) {
this.sb = sb;
- this.supportedProtocols = new HashSet<String>(Arrays.asList(new String[]{"http","https","ftp","smb","file"}));
+ this.supportedProtocols = new HashSet<>(Arrays.asList("http", "https", "ftp", "smb", "file"));
// initiate loader objects
this.httpLoader = new HTTPLoader(sb, LoaderDispatcher.log);
this.ftpLoader = new FTPLoader(sb, LoaderDispatcher.log);
this.smbLoader = new SMBLoader(sb, LoaderDispatcher.log);
this.fileLoader = new FileLoader(sb, LoaderDispatcher.log);
- this.loaderSteering = new ConcurrentHashMap<DigestURL, Semaphore>();
+ this.loaderSteering = new ConcurrentHashMap<>();
}
public boolean isSupportedProtocol(final String protocol) {
@@ -115,7 +115,7 @@ public final class LoaderDispatcher {
final boolean forText,
final boolean global
) {
- CrawlProfile profile =
+ final CrawlProfile profile =
(forText) ?
((global) ?
this.sb.crawler.defaultTextSnippetGlobalProfile :
@@ -137,7 +137,7 @@ public final class LoaderDispatcher {
public void load(final DigestURL url, final CacheStrategy cacheStratgy, final int maxFileSize, final File targetFile, BlacklistType blacklistType, ClientIdentification.Agent agent) throws IOException {
- final byte[] b = load(request(url, false, true), cacheStratgy, maxFileSize, blacklistType, agent).getContent();
+ final byte[] b = this.load(this.request(url, false, true), cacheStratgy, maxFileSize, blacklistType, agent).getContent();
if (b == null) throw new IOException("load == null");
final File tmp = new File(targetFile.getAbsolutePath() + ".tmp");
@@ -149,7 +149,7 @@ public final class LoaderDispatcher {
}
public Response load(final Request request, final CacheStrategy cacheStrategy, final BlacklistType blacklistType, ClientIdentification.Agent agent) throws IOException {
- return load(request, cacheStrategy, protocolMaxFileSize(request.url()), blacklistType, agent);
+ return this.load(request, cacheStrategy, this.protocolMaxFileSize(request.url()), blacklistType, agent);
}
/**
@@ -169,7 +169,7 @@ public final class LoaderDispatcher {
if (check != null && cacheStrategy != CacheStrategy.NOCACHE) {
// a loading process is going on for that url
//ConcurrentLog.info("LoaderDispatcher", "waiting for " + request.url().toNormalform(true));
- long t = System.currentTimeMillis();
+ final long t = System.currentTimeMillis();
try { check.tryAcquire(5, TimeUnit.SECONDS);} catch (final InterruptedException e) {}
ConcurrentLog.info("LoaderDispatcher", "waited " + (System.currentTimeMillis() - t) + " ms for " + request.url().toNormalform(true));
// now the process may have terminated and we run a normal loading
@@ -178,7 +178,7 @@ public final class LoaderDispatcher {
this.loaderSteering.put(request.url(), new Semaphore(0));
try {
- final Response response = loadInternal(request, cacheStrategy, maxFileSize, blacklistType, agent);
+ final Response response = this.loadInternal(request, cacheStrategy, maxFileSize, blacklistType, agent);
// finally block cleans up loaderSteering and semaphore
return response;
} catch (final IOException e) {
@@ -215,7 +215,7 @@ public final class LoaderDispatcher {
}
// check if we have the page in the cache
- Response response = loadFromCache(request, cacheStrategy, agent, url, crawlProfile);
+ Response response = this.loadFromCache(request, cacheStrategy, agent, url, crawlProfile);
if(response != null) {
return response;
}
@@ -230,7 +230,7 @@ public final class LoaderDispatcher {
// check access time: this is a double-check (we checked possibly already in the balancer)
// to make sure that we don't DoS the target by mistake
- checkAccessTime(agent, url);
+ this.checkAccessTime(agent, url);
// now it's for sure that we will access the target. Remember the access time
if (host != null) {
@@ -305,7 +305,7 @@ public final class LoaderDispatcher {
// create request header values and a response object because we need that
// in case that we want to return the cached content in the next step
final RequestHeader requestHeader = new RequestHeader();
- requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent);
+ requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent());
String refererURL = null;
if (request.referrerhash() != null) refererURL = this.sb.getURL(request.referrerhash());
if (refererURL != null) requestHeader.put(RequestHeader.REFERER, refererURL);
@@ -373,7 +373,7 @@ public final class LoaderDispatcher {
}
// check if we have the page in the cache
- Response cachedResponse = loadFromCache(request, cacheStrategy, agent, url, crawlProfile);
+ final Response cachedResponse = this.loadFromCache(request, cacheStrategy, agent, url, crawlProfile);
if (cachedResponse != null) {
return new StreamResponse(cachedResponse, new ByteArrayInputStream(cachedResponse.getContent()));
}
@@ -388,7 +388,7 @@ public final class LoaderDispatcher {
// check access time: this is a double-check (we checked possibly already in the balancer)
// to make sure that we don't DoS the target by mistake
- checkAccessTime(agent, url);
+ this.checkAccessTime(agent, url);
// now it's for sure that we will access the target. Remember the access time
if (host != null) {
@@ -422,18 +422,18 @@ public final class LoaderDispatcher {
*/
private void checkAccessTime(ClientIdentification.Agent agent, final DigestURL url) {
if (!url.isLocal()) {
- String host = url.getHost();
+ final String host = url.getHost();
final Long lastAccess = accessTime.get(host);
long wait = 0;
if (lastAccess != null)
- wait = Math.max(0, agent.minimumDelta + lastAccess.longValue() - System.currentTimeMillis());
+ wait = Math.max(0, agent.minimumDelta() + lastAccess.longValue() - System.currentTimeMillis());
if (wait > 0) {
// force a sleep here. Instead just sleep we clean up the
// accessTime map
final long untilTime = System.currentTimeMillis() + wait;
cleanupAccessTimeTable(untilTime);
if (System.currentTimeMillis() < untilTime) {
- long frcdslp = untilTime - System.currentTimeMillis();
+ final long frcdslp = untilTime - System.currentTimeMillis();
LoaderDispatcher.log.info("Forcing sleep of " + frcdslp + " ms for host " + host);
try {
Thread.sleep(frcdslp);
@@ -474,7 +474,7 @@ public final class LoaderDispatcher {
*/
public byte[] loadContent(final Request request, final CacheStrategy cacheStrategy, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException {
// try to download the resource using the loader
- final Response entry = load(request, cacheStrategy, blacklistType, agent);
+ final Response entry = this.load(request, cacheStrategy, blacklistType, agent);
if (entry == null) return null; // not found in web
// read resource body (if it is there)
@@ -498,7 +498,7 @@ public final class LoaderDispatcher {
Semaphore check = this.loaderSteering.get(request.url());
if (check != null && cacheStrategy != CacheStrategy.NOCACHE) {
// a loading process is going on for that url
- long t = System.currentTimeMillis();
+ final long t = System.currentTimeMillis();
try {
check.tryAcquire(5, TimeUnit.SECONDS);
} catch (final InterruptedException e) {
@@ -511,8 +511,8 @@ public final class LoaderDispatcher {
this.loaderSteering.put(request.url(), new Semaphore(0));
try {
- response = openInputStreamInternal(request, cacheStrategy, maxFileSize, blacklistType, agent);
- } catch(IOException ioe) {
+ response = this.openInputStreamInternal(request, cacheStrategy, maxFileSize, blacklistType, agent);
+ } catch(final IOException ioe) {
/* Do not re encapsulate any eventual IOException in an IOException */
throw ioe;
} catch (final Throwable e) {
@@ -539,14 +539,14 @@ public final class LoaderDispatcher {
*/
public StreamResponse openInputStream(final Request request, final CacheStrategy cacheStrategy,
BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException {
- final int maxFileSize = protocolMaxFileSize(request.url());
+ final int maxFileSize = this.protocolMaxFileSize(request.url());
return this.openInputStream(request, cacheStrategy, blacklistType, agent, maxFileSize);
}
public Document[] loadDocuments(final Request request, final CacheStrategy cacheStrategy, final int maxFileSize, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException, Parser.Failure {
// load resource
- final Response response = load(request, cacheStrategy, maxFileSize, blacklistType, agent);
+ final Response response = this.load(request, cacheStrategy, maxFileSize, blacklistType, agent);
final DigestURL url = request.url();
if (response == null) throw new IOException("no Response for url " + url);
@@ -554,11 +554,11 @@ public final class LoaderDispatcher {
if (response.getContent() == null || response.getResponseHeader() == null) throw new IOException("no Content available for url " + url);
// parse resource
- Document[] documents = response.parse();
+ final Document[] documents = response.parse();
- String x_robots_tag = response.getResponseHeader().getXRobotsTag();
+ final String x_robots_tag = response.getResponseHeader().getXRobotsTag();
if (x_robots_tag.indexOf("noindex",0) >= 0) {
- for (Document d: documents) d.setIndexingDenied(true);
+ for (final Document d: documents) d.setIndexingDenied(true);
}
return documents;
@@ -566,7 +566,7 @@ public final class LoaderDispatcher {
public Document loadDocument(final DigestURL location, final CacheStrategy cachePolicy, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException {
// load resource
- Request request = request(location, true, false);
+ final Request request = this.request(location, true, false);
final Response response = this.load(request, cachePolicy, blacklistType, agent);
final DigestURL url = request.url();
if (response == null) throw new IOException("no Response for url " + url);
@@ -576,10 +576,10 @@ public final class LoaderDispatcher {
// parse resource
try {
- Document[] documents = response.parse();
- Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);
+ final Document[] documents = response.parse();
+ final Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);
- String x_robots_tag = response.getResponseHeader().getXRobotsTag();
+ final String x_robots_tag = response.getResponseHeader().getXRobotsTag();
if (x_robots_tag.indexOf("noindex",0) >= 0) merged.setIndexingDenied(true);
return merged;
@@ -600,7 +600,7 @@ public final class LoaderDispatcher {
public Document loadDocumentAsStream(final DigestURL location, final CacheStrategy cachePolicy,
final BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException {
// load resource
- Request request = request(location, true, false);
+ final Request request = this.request(location, true, false);
final StreamResponse streamResponse = this.openInputStream(request, cachePolicy, blacklistType, agent);
final Response response = streamResponse.getResponse();
final DigestURL url = request.url();
@@ -613,10 +613,10 @@ public final class LoaderDispatcher {
// parse resource
try {
- Document[] documents = streamResponse.parse();
- Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);
+ final Document[] documents = streamResponse.parse();
+ final Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);
- String x_robots_tag = response.getResponseHeader().getXRobotsTag();
+ final String x_robots_tag = response.getResponseHeader().getXRobotsTag();
if (x_robots_tag.indexOf("noindex",0) >= 0) {
merged.setIndexingDenied(true);
}
@@ -659,7 +659,7 @@ public final class LoaderDispatcher {
public Document loadDocumentAsLimitedStream(final DigestURL location, final CacheStrategy cachePolicy,
final BlacklistType blacklistType, final ClientIdentification.Agent agent, final int maxLinks, final long maxBytes) throws IOException {
// load resource
- Request request = request(location, true, false);
+ final Request request = this.request(location, true, false);
final StreamResponse streamResponse = this.openInputStream(request, cachePolicy, blacklistType, agent, -1);
final Response response = streamResponse.getResponse();
final DigestURL url = request.url();
@@ -672,10 +672,10 @@ public final class LoaderDispatcher {
// parse resource
try {
- Document[] documents = streamResponse.parseWithLimits(maxLinks, maxBytes);
- Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);
+ final Document[] documents = streamResponse.parseWithLimits(maxLinks, maxBytes);
+ final Document merged = Document.mergeDocuments(location, response.getMimeType(), documents);
- String x_robots_tag = response.getResponseHeader().getXRobotsTag();
+ final String x_robots_tag = response.getResponseHeader().getXRobotsTag();
if (x_robots_tag.indexOf("noindex",0) >= 0) {
merged.setIndexingDenied(true);
}
@@ -699,7 +699,7 @@ public final class LoaderDispatcher {
BlacklistType blacklistType,
final ClientIdentification.Agent agent,
final int timezoneOffset) throws IOException {
- final Response response = load(request(url, true, false), cacheStrategy, Integer.MAX_VALUE, blacklistType, agent);
+ final Response response = this.load(this.request(url, true, false), cacheStrategy, Integer.MAX_VALUE, blacklistType, agent);
if (response == null) throw new IOException("response == null");
final ResponseHeader responseHeader = response.getResponseHeader();
if (response.getContent() == null) throw new IOException("resource == null");
@@ -769,7 +769,7 @@ public final class LoaderDispatcher {
if (this.cache != null && this.cache.exists()) return;
try {
// load from the net
- final Response response = load(request(this.url, false, true), this.cacheStrategy, this.maxFileSize, this.blacklistType, this.agent);
+ final Response response = LoaderDispatcher.this.load(LoaderDispatcher.this.request(this.url, false, true), this.cacheStrategy, this.maxFileSize, this.blacklistType, this.agent);
final byte[] b = response.getContent();
if (this.cache != null) FileUtils.copy(b, this.cache);
} catch (final MalformedURLException e) {} catch (final IOException e) {}
diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java
index 073aec83c..edeaf7e67 100644
--- a/source/net/yacy/server/serverSwitch.java
+++ b/source/net/yacy/server/serverSwitch.java
@@ -690,7 +690,7 @@ public class serverSwitch {
netdef = netdef.trim();
try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
final RequestHeader reqHeader = new RequestHeader();
- reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);
+ reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent());
client.setHeader(reqHeader.entrySet());
final byte[] data = client.GETbytes(uri,
this.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),