Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
to prevent crawler to concurrently access and alter same crawl queue after restart, put hosthash in queue's filename (which is used as primary key for crawl queue. Hint: initial hosthash from url and recalculated hosthash from just hostname:port are not the same. fixes http://mantis.tokeek.de/view.php?id=668 (partially)
reger22db449f2ab8
diff --git a/source/net/yacy/crawler/HostBalancer.java b/source/net/yacy/crawler/HostBalancer.javaindex 9ffe4cf23..b6338b6c1 100644--- a/source/net/yacy/crawler/HostBalancer.java+++ b/source/net/yacy/crawler/HostBalancer.java@@ -103,7 +103,7 @@ public class HostBalancer implements Balancer {queue.close();FileUtils.deletedelete(queuePath);} else {- queues.put(DigestURL.hosthash(queue.getHost(), queue.getPort()), queue);+ queues.put(queue.getHostHash(), queue);}} catch (MalformedURLException | RuntimeException e) {log.warn("delete queue due to init error for " + hostsPath.getName() + " host=" + hoststr + " " + e.getLocalizedMessage());@@ -244,11 +244,11 @@ public class HostBalancer implements Balancer {public String push(final Request entry, CrawlProfile profile, final RobotsTxt robots) throws IOException, SpaceExceededException {if (this.has(entry.url().hash())) return "double occurrence";depthCache.put(entry.url().hash(), entry.depth());- String hosthash = ASCII.String(entry.url().hash(), 6, 6);+ String hosthash = entry.url().hosthash();synchronized (this) {HostQueue queue = this.queues.get(hosthash);if (queue == null) {- queue = new HostQueue(this.hostsPath, entry.url().getHost(), entry.url().getPort(), this.queues.size() > this.onDemandLimit, this.exceed134217727);+ queue = new HostQueue(this.hostsPath, entry.url(), this.queues.size() > this.onDemandLimit, this.exceed134217727);this.queues.put(hosthash, queue);// profile might be null when continue crawls after YaCy restartrobots.ensureExist(entry.url(), profile == null ? ClientIdentification.yacyInternetCrawlerAgent : profile.getAgent(), true); // concurrently load all robots.txt@@ -376,7 +376,7 @@ public class HostBalancer implements Balancer {for (String h: lastEntries) this.roundRobinHostHashes.remove(h);}}-+/*// first strategy: get one entry which does not need sleep timeIterator<String> nhhi = this.roundRobinHostHashes.iterator();@@ -386,7 +386,7 @@ public class HostBalancer implements Balancer {if (rhq == null) {nhhi.remove();continue nosleep;- }+ }int delta = Latency.waitingRemainingGuessed(rhq.getHost(), rhh, robots, ClientIdentification.yacyInternetCrawlerAgent);if (delta <= 10 || this.roundRobinHostHashes.size() == 1 || rhq.size() == 1) {nhhi.remove();@@ -494,7 +494,7 @@ public class HostBalancer implements Balancer {map.put(hq.getHost() + ":" + hq.getPort(), new Integer[]{hq.size(), delta});} catch (MalformedURLException e) {ConcurrentLog.logException(e);- }+ }return map;}diff --git a/source/net/yacy/crawler/HostQueue.java b/source/net/yacy/crawler/HostQueue.javaindex d16a65f39..3e9621a11 100644--- a/source/net/yacy/crawler/HostQueue.java+++ b/source/net/yacy/crawler/HostQueue.java@@ -64,28 +64,48 @@ public class HostQueue implements Balancer {private static final int EcoFSBufferSize = 1000;private static final int objectIndexBufferSize = 1000;- private final File hostPath;+ private final File hostPath; // path to the stack filesprivate final String hostName;- private String hostHash;+ private final String hostHash;private final int port;private final boolean exceed134217727;private final boolean onDemand;private TreeMap<Integer, Index> depthStacks;+ /**+ * Create or open host queue. The host part of the hostUrl parameter is used+ * to calculate the stack directory name.+ *+ * @param hostsPath+ * @param hostUrl+ * @param onDemand+ * @param exceed134217727+ * @throws MalformedURLException+ */public HostQueue (final File hostsPath,- final String hostName,- final int port,+ final DigestURL hostUrl, // any url from that host (only host data are extracted)final boolean onDemand,final boolean exceed134217727) throws MalformedURLException {this.onDemand = onDemand;this.exceed134217727 = exceed134217727;- this.hostName = (hostName == null) ? "localhost" : hostName; // might be null (file://) but hostqueue needs a name (for queue file)- this.port = port;- this.hostPath = new File(hostsPath, this.hostName + "." + this.port);+ this.hostName = (hostUrl.getHost() == null) ? "localhost" : hostUrl.getHost(); // might be null (file://) but hostqueue needs a name (for queue file)+ this.port = hostUrl.getPort();+ this.hostHash = hostUrl.hosthash(); // hosthash is calculated by protocol + hostname + port+ // hostName/port included just for human readability (& historically), "-#" marker used to define begin of hosthash in directoryname+ this.hostPath = new File(hostsPath, this.hostName + "-#"+ this.hostHash + "." + this.port);init();}+ /**+ * Initializes host queue from cache files. The internal id of the queue is+ * extracted form the path name an must match the key initially generated+ * currently the hosthash is used as id.+ * @param hostPath path of the stack directory (containing the primary key/id of the queue)+ * @param onDemand+ * @param exceed134217727+ * @throws MalformedURLException+ */public HostQueue (final File hostPath,final boolean onDemand,@@ -95,10 +115,14 @@ public class HostQueue implements Balancer {this.hostPath = hostPath;// parse the hostName and port from the file nameString filename = hostPath.getName();- int p = filename.lastIndexOf('.');- if (p < 0) throw new RuntimeException("hostPath name must contain a dot: " + filename);- this.hostName = filename.substring(0, p);- this.port = Integer.parseInt(filename.substring(p + 1)); // consider "host.com" contains dot but no required port -> will throw exception+ int pdot = filename.lastIndexOf('.');+ if (pdot < 0) throw new RuntimeException("hostPath name must contain a dot: " + filename);+ this.port = Integer.parseInt(filename.substring(pdot + 1)); // consider "host.com" contains dot but no required port -> will throw exception+ int p1 = filename.lastIndexOf("-#");+ if (p1 >= 0) {+ this.hostName = filename.substring(0,p1);+ this.hostHash = filename.substring(p1+2,pdot);+ } else throw new RuntimeException("hostPath name must contain -# followd by hosthash: " + filename);init();}@@ -107,14 +131,6 @@ public class HostQueue implements Balancer {* @throws MalformedURLException if directory for the host could not be created*/private final void init() throws MalformedURLException {- try {- if (this.hostName == null)- this.hostHash="";- else- this.hostHash = DigestURL.hosthash(this.hostName, this.port);- } catch (MalformedURLException e) {- this.hostHash = "";- }if (!(this.hostPath.exists())) {this.hostPath.mkdirs();if (!this.hostPath.exists()) { // check if directory created (if not, likely a name violation)@@ -134,6 +150,15 @@ public class HostQueue implements Balancer {return this.port;}+ /**+ * Get the hosthash of this queue determined during init.+ *+ * @return+ */+ public String getHostHash() {+ return this.hostHash;+ }+private int openAllStacks() {String[] l = this.hostPath.list();int c = 0;@@ -157,26 +182,6 @@ public class HostQueue implements Balancer {return c;}- public synchronized int getLowestStackDepth() {- while (this.depthStacks.size() > 0) {- Map.Entry<Integer, Index> entry;- synchronized (this) {- entry = this.depthStacks.firstEntry();- }- if (entry == null) return 0; // happens only if map is empty- if (entry.getValue().size() == 0) {- entry.getValue().close();- deletedelete(getFile(entry.getKey()));- this.depthStacks.remove(entry.getKey());- continue;- }- return entry.getKey();- }- // this should not happen but it happens if a deletion is done- //assert false;- return 0;- }-private Index getLowestStack() {while (this.depthStacks.size() > 0) {Map.Entry<Integer, Index> entry;@@ -196,16 +201,17 @@ public class HostQueue implements Balancer {//assert false;return null;}-++ /**+ * Get existing url stack with crawl depth or create a new (empty) stack+ *+ * @param depth+ * @return existing or new/empty stack+ */private Index getStack(int depth) {Index depthStack;- synchronized (this) {- depthStack = this.depthStacks.get(depth);- if (depthStack != null) return depthStack;- }// create a new stacksynchronized (this) {- // check againdepthStack = this.depthStacks.get(depth);if (depthStack != null) return depthStack;// now actually create a new stack