Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Fixed shutdown for crawler.MaxActiveThreads value greater than 200
lucciomandcdea2d02f81
diff --git a/source/net/yacy/crawler/data/CrawlQueues.java b/source/net/yacy/crawler/data/CrawlQueues.javaindex 025b5b112..35c244df1 100644--- a/source/net/yacy/crawler/data/CrawlQueues.java+++ b/source/net/yacy/crawler/data/CrawlQueues.java@@ -88,7 +88,9 @@ public class CrawlQueues {this.sb = sb;final int maxWorkers = (int) sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10);this.worker = new Loader[maxWorkers];- this.workerQueue = new ArrayBlockingQueue<Request>(200);+ /* We initialize workerQueue with the same capacity as worker array, because this same queue+ * will be used to send POISON_REQUEST items consumed by all eventually running workers in the close() function*/+ this.workerQueue = new ArrayBlockingQueue<Request>(maxWorkers);this.remoteCrawlProviderHashes = null;// start crawling management@@ -128,11 +130,19 @@ public class CrawlQueues {}public synchronized void close() {+ /* We close first the noticeURL because it is used to fill the workerQueue.*/+ this.noticeURL.close();// removed pending requeststhis.workerQueue.clear();// wait for all workers to finish- for (int i = 0; i < this.worker.length; i++) {- try {this.workerQueue.put(POISON_REQUEST);} catch (InterruptedException e) {}+ for (int i = 0; i < this.workerQueue.remainingCapacity(); i++) {+ /* We use the non-blocking offer() function instead of the blocking put() in the unlikely eventual case another thread+ * added an element to workerQueue during this loop.*/+ try {+ this.workerQueue.offer(POISON_REQUEST, 1, TimeUnit.SECONDS);+ } catch (InterruptedException e) {+ CrawlQueues.log.warn("Interrupted while adding POISON_REQUEST to the workerQueue");+ }}for (final Loader w: this.worker) {if (w != null && w.isAlive()) {@@ -140,11 +150,10 @@ public class CrawlQueues {w.join(1000);if (w.isAlive()) w.interrupt();} catch (final InterruptedException e) {- ConcurrentLog.logException(e);+ CrawlQueues.log.warn("Interrupted while waiting for worker termination.");}}}- this.noticeURL.close();if (this.delegatedURL != null) this.delegatedURL.clear();}