Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Removed some unnecessary uses of java.lang.reflect api.
luccioman4eba88f2ff14
diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.javaindex 2334ea074..91ea5abde 100644--- a/htroot/ConfigBasic.java+++ b/htroot/ConfigBasic.java@@ -35,12 +35,11 @@ import java.util.List;import java.util.regex.Pattern;import net.yacy.cora.protocol.Domains;-import net.yacy.cora.protocol.HeaderFramework;import net.yacy.cora.protocol.RequestHeader;import net.yacy.data.Translator;import net.yacy.data.WorkTables;import net.yacy.http.YaCyHttpServer;-import net.yacy.kelondro.workflow.InstantBusyThread;+import net.yacy.peers.OnePeerPingBusyThread;import net.yacy.peers.Seed;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;@@ -48,8 +47,8 @@ import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;import net.yacy.server.http.HTTPDFileHandler;import net.yacy.utils.translation.TranslatorXliff;-import net.yacy.utils.upnp.UPnPMappingType;import net.yacy.utils.upnp.UPnP;+import net.yacy.utils.upnp.UPnPMappingType;public class ConfigBasic {@@ -58,7 +57,7 @@ public class ConfigBasic {private static final int NEXTSTEP_PEERNAME = 2;private static final int NEXTSTEP_PEERPORT = 3;private static final int NEXTSTEP_RECONNECT = 4;-+public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException {// return variable that accumulates replacements@@ -81,7 +80,7 @@ public class ConfigBasic {//boolean doPeerPing = false;if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) {- InstantBusyThread.oneTimeJob(sb.yc, "peerPing", 0);+ new OnePeerPingBusyThread(sb.yc).start();//doPeerPing = true;}diff --git a/htroot/api/ymarks/import_ymark.java b/htroot/api/ymarks/import_ymark.javaindex 5fa57b101..5344539f9 100644--- a/htroot/api/ymarks/import_ymark.java+++ b/htroot/api/ymarks/import_ymark.java@@ -33,7 +33,7 @@ import net.yacy.data.ymark.YMarkUtil;import net.yacy.data.ymark.YMarkXBELImporter;import net.yacy.document.Parser.Failure;import net.yacy.kelondro.blob.Tables;-import net.yacy.kelondro.workflow.InstantBusyThread;+import net.yacy.kelondro.workflow.OneTimeBusyThread;import net.yacy.search.Switchboard;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;@@ -71,7 +71,7 @@ public class import_ymark {empty = true;}YMarkAutoTagger autoTagger = new YMarkAutoTagger(autoTaggingQueue, sb.loader, sb.tables.bookmarks, bmk_user, merge);- InstantBusyThread.oneTimeJob(autoTagger, 0);+ OneTimeBusyThread.startFromRunnable(autoTagger, 0);}if(isAdmin && post.containsKey("table") && post.get("table").length() > 0) {@@ -109,8 +109,8 @@ public class import_ymark {MonitoredReader reader = new MonitoredReader(new InputStreamReader(stream, StandardCharsets.UTF_8), 1024*16, bytes.length);if(post.get("importer").equals("html") && reader != null) {final YMarkHTMLImporter htmlImporter = new YMarkHTMLImporter(reader, queueSize, root);- InstantBusyThread.oneTimeJob(htmlImporter, 0);- InstantBusyThread.oneTimeJob(htmlImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);+ OneTimeBusyThread.startFromRunnable(htmlImporter, 0);+ OneTimeBusyThread.startFromRunnable(htmlImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);prop.put("status", "1");} else if(post.get("importer").equals("xbel") && reader != null) {final YMarkXBELImporter xbelImporter;@@ -123,13 +123,13 @@ public class import_ymark {prop.put("status", "0");return prop;}- InstantBusyThread.oneTimeJob(xbelImporter, 0);- InstantBusyThread.oneTimeJob(xbelImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);+ OneTimeBusyThread.startFromRunnable(xbelImporter, 0);+ OneTimeBusyThread.startFromRunnable(xbelImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);prop.put("status", "1");} else if(post.get("importer").equals("json") && reader != null) {YMarkJSONImporter jsonImporter;jsonImporter = new YMarkJSONImporter(reader, queueSize, root);- InstantBusyThread.oneTimeJob(jsonImporter, 0);+ OneTimeBusyThread.startFromRunnable(jsonImporter, 0);while ((bmk = jsonImporter.take()) != YMarkEntry.POISON) {putBookmark(sb, bmk_user, bmk, autoTaggingQueue, autotag, empty, indexing, medialink);}@@ -207,8 +207,8 @@ public class import_ymark {mreader.addChangeListener(sb.tables.bookmarks.getProgressListener("DMOZImporter"));DMOZImporter.setDepth(6);- InstantBusyThread.oneTimeJob(DMOZImporter, 0);- InstantBusyThread.oneTimeJob(DMOZImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);+ OneTimeBusyThread.startFromRunnable(DMOZImporter, 0);+ OneTimeBusyThread.startFromRunnable(DMOZImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0);prop.put("status", "1");} catch (final Exception e) {diff --git a/source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.java b/source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.javaindex 8f82b8e42..9a3d12e2f 100644--- a/source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.java+++ b/source/net/yacy/contentcontrol/ContentControlFilterUpdateThread.java@@ -8,7 +8,7 @@ import net.yacy.kelondro.blob.Tables.Row;import net.yacy.repository.FilterEngine;import net.yacy.search.Switchboard;-public class ContentControlFilterUpdateThread {+public class ContentControlFilterUpdateThread implements Runnable {private final Switchboard sb;@@ -22,6 +22,7 @@ public class ContentControlFilterUpdateThread {}+ @Overridepublic final void run() {if (!this.locked) {diff --git a/source/net/yacy/contentcontrol/SMWListSyncThread.java b/source/net/yacy/contentcontrol/SMWListSyncThread.javaindex 805aa6279..80cf8a0d3 100644--- a/source/net/yacy/contentcontrol/SMWListSyncThread.java+++ b/source/net/yacy/contentcontrol/SMWListSyncThread.java@@ -13,7 +13,7 @@ import net.yacy.cora.util.CommonPattern;import net.yacy.cora.util.ConcurrentLog;import net.yacy.search.Switchboard;-public class SMWListSyncThread {+public class SMWListSyncThread implements Runnable {private final Switchboard sb;private Boolean locked = false;@@ -55,6 +55,7 @@ public class SMWListSyncThread {return ret;}+ @Overridepublic final void run() {if (!this.locked) {diff --git a/source/net/yacy/kelondro/workflow/InstantBusyThread.java b/source/net/yacy/kelondro/workflow/InstantBusyThread.javaindex e8d62ab23..ca091c173 100644--- a/source/net/yacy/kelondro/workflow/InstantBusyThread.java+++ b/source/net/yacy/kelondro/workflow/InstantBusyThread.java@@ -24,77 +24,36 @@package net.yacy.kelondro.workflow;-import java.lang.reflect.InvocationTargetException;-import java.lang.reflect.Method;import java.util.TreeMap;import net.yacy.cora.util.ConcurrentLog;-public final class InstantBusyThread extends AbstractBusyThread implements BusyThread {+public abstract class InstantBusyThread extends AbstractBusyThread implements BusyThread {- private Method jobExecMethod, jobCountMethod, freememExecMethod;- private final Object environment;private final Long handle;private static final TreeMap<Long, String> jobs = new TreeMap<Long, String>();- public InstantBusyThread(- final Object env,- final String jobExec,- final String jobCount,- final String freemem,- final long idleSleep,- final long busySleep) {+ /**+ * @param idleSleep defines min idle sleep time that can be set via setIdleSleep()+ * @param busySleep defines min busy sleep time that can be set via setBusySleep()+ */+ public InstantBusyThread(final long idleSleep, final long busySleep) {+ this("InstantBusyThread.job", idleSleep, busySleep);+ }++ /**+ * @param jobName the job name used to monitor the thread+ * @param idleSleep defines min idle sleep time that can be set via setIdleSleep()+ * @param busySleep defines min busy sleep time that can be set via setBusySleep()+ */+ public InstantBusyThread(final String jobName, final long idleSleep, final long busySleep) {super(idleSleep, busySleep);-- // jobExec is the name of a method of the object 'env' that executes the one-step-run- // jobCount is the name of a method that returns the size of the job- // freemem is the name of a method that tries to free memory and returns void- final Class<?> theClass = (env instanceof Class<?>) ? (Class<?>) env : env.getClass();- try {- this.jobExecMethod = theClass.getMethod(jobExec);- } catch (final NoSuchMethodException e) {- throw new RuntimeException("serverInstantThread, wrong declaration of jobExec: " + e.getMessage());- }- try {- if (jobCount == null)- this.jobCountMethod = null;- else- this.jobCountMethod = theClass.getMethod(jobCount);-- } catch (final NoSuchMethodException e) {- throw new RuntimeException("serverInstantThread, wrong declaration of jobCount: " + e.getMessage());- }- try {- if (freemem == null)- this.freememExecMethod = null;- else- this.freememExecMethod = theClass.getMethod(freemem);-- } catch (final NoSuchMethodException e) {- throw new RuntimeException("serverInstantThread, wrong declaration of freemem: " + e.getMessage());- }- this.environment = (env instanceof Class<?>) ? null : env;- setName("BusyThread " + theClass.getName() + "." + jobExec);+ setName("BusyThread " + jobName);this.handle = Long.valueOf(System.currentTimeMillis() + getName().hashCode());}-- @Override- public int getJobCount() {- if (this.jobCountMethod == null) return Integer.MAX_VALUE;- try {- final Object result = this.jobCountMethod.invoke(this.environment);- return (result instanceof Integer) ? ((Integer) result).intValue() : -1;- } catch (final IllegalAccessException e) {- return -1;- } catch (final IllegalArgumentException e) {- return -1;- } catch (final InvocationTargetException e) {- ConcurrentLog.severe("BUSYTHREAD", "invocation serverInstantThread of thread '" + getName() + "': " + e.getMessage(), e);- return -1;- }- }+@Overridepublic boolean job() throws Exception {@@ -102,25 +61,13 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyTsynchronized(jobs) {jobs.put(this.handle, getName());}boolean jobHasDoneSomething = false;try {- final Object result = this.jobExecMethod.invoke(this.environment);- if (result == null) jobHasDoneSomething = true;- else if (result instanceof Boolean) jobHasDoneSomething = ((Boolean) result).booleanValue();- } catch (final IllegalAccessException e) {- ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.job: " + e.getMessage());- ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'");- terminate(false);+ jobHasDoneSomething = jobImpl();} catch (final IllegalArgumentException e) {- ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.job: " + e.getMessage());+ ConcurrentLog.severe("BUSYTHREAD", "Internal Error in InstantBusyThread.job: " + e.getMessage());ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'");terminate(false);- } catch (final InvocationTargetException e) {- final String targetException = e.getTargetException().getMessage();- ConcurrentLog.logException(e);- ConcurrentLog.logException(e.getCause());- ConcurrentLog.logException(e.getTargetException());- ConcurrentLog.severe("BUSYTHREAD", "Runtime Error in serverInstantThread.job, thread '" + getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException());} catch (final OutOfMemoryError e) {- ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in serverInstantThread.job, thread '" + getName() + "': " + e.getMessage());+ ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in InstantBusyThread.job, thread '" + getName() + "': " + e.getMessage());ConcurrentLog.logException(e);freemem();} catch (final Exception e) {@@ -130,46 +77,36 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyTsynchronized(jobs) {jobs.remove(this.handle);}return jobHasDoneSomething;}++ /**+ * The job's main logic implementation+ * @return true if it has done something, false if it is idle and does not expect to work on more for a longer time+ * @throws Exception when an unexpected error occurred+ */+ public abstract boolean jobImpl() throws Exception;@Overridepublic void freemem() {- if (this.freememExecMethod == null) return;try {- this.freememExecMethod.invoke(this.environment);- } catch (final IllegalAccessException e) {- ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.freemem: " + e.getMessage());- ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'");- terminate(false);- } catch (final IllegalArgumentException e) {- ConcurrentLog.severe("BUSYTHREAD", "Internal Error in serverInstantThread.freemem: " + e.getMessage());- ConcurrentLog.severe("BUSYTHREAD", "shutting down thread '" + getName() + "'");- terminate(false);- } catch (final InvocationTargetException e) {- final String targetException = e.getTargetException().getMessage();- if (targetException.indexOf("heap space",0) > 0) ConcurrentLog.logException(e.getTargetException());- ConcurrentLog.severe("BUSYTHREAD", "Runtime Error in serverInstantThread.freemem, thread '" + getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException());- ConcurrentLog.logException(e.getTargetException());+ freememImpl();} catch (final OutOfMemoryError e) {- ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in serverInstantThread.freemem, thread '" + getName() + "': " + e.getMessage());+ ConcurrentLog.severe("BUSYTHREAD", "OutOfMemory Error in InstantBusyThread.freemem, thread '" + getName() + "': " + e.getMessage());ConcurrentLog.logException(e);}}++ @Override+ public int getJobCount() {+ return Integer.MAX_VALUE;+ }++ /**+ * Called when an outOfMemoryCycle is performed.+ */+ public void freememImpl() {+ // Do nothing in this implementation, please override+ }- public static BusyThread oneTimeJob(final Object env, final String jobExec, final long startupDelay) {- // start the job and execute it once as background process- final BusyThread thread = new InstantBusyThread(env, jobExec, null, null, Long.MIN_VALUE, Long.MIN_VALUE);- thread.setStartupSleep(startupDelay);- thread.setIdleSleep(-1);- thread.setBusySleep(-1);- thread.setMemPreReqisite(0);- thread.setLoadPreReqisite(Double.MAX_VALUE); // this is called during initialization phase and some code parts depend on it; therefore we cannot set a prerequisite that prevents the start of that thread- thread.start();- return thread;- }-- public static WorkflowThread oneTimeJob(final Runnable thread, final long startupDelay) {- return oneTimeJob(thread, "run", startupDelay);- }@Overridepublic void open() {@@ -180,5 +117,35 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyTpublic synchronized void close() {// Not implemented in this thread}++ /**+ * Construct an InstantBusyThread instance from a runnable task.+ *+ * @param task+ * the task to run as a job+ * @param idleSleep+ * defines min idle sleep time that can be set via setIdleSleep()+ * @param busySleep+ * defines min busy sleep time that can be set via setBusySleep()+ * @return a InstantBusyThread instance+ * @throws IllegalArgumentException+ * when the task is null+ */+ public static InstantBusyThread createFromRunnable(final Runnable task, final long idleSleep,+ final long busySleep) {+ if (task == null) {+ throw new IllegalArgumentException("Runnable task must not be null");+ }+ InstantBusyThread busyThread = new InstantBusyThread(task.getClass().getName() + ".run", idleSleep, busySleep) {++ @Override+ public boolean jobImpl() throws Exception {+ task.run();+ return true;+ }++ };+ return busyThread;+ }}diff --git a/source/net/yacy/kelondro/workflow/OneTimeBusyThread.java b/source/net/yacy/kelondro/workflow/OneTimeBusyThread.javanew file mode 100644index 000000000..d7bbf832e--- /dev/null+++ b/source/net/yacy/kelondro/workflow/OneTimeBusyThread.java@@ -0,0 +1,80 @@+// OneTimeBusyThread.java+// ---------------------------+// Copyright 2017 by luccioman; https://github.com/luccioman+//+// This is a part of YaCy, a peer-to-peer based web search engine+//+// LICENSE+//+// This program is free software; you can redistribute it and/or modify+// it under the terms of the GNU General Public License as published by+// the Free Software Foundation; either version 2 of the License, or+// (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU General Public License for more details.+//+// You should have received a copy of the GNU General Public License+// along with this program; if not, write to the Free Software+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA++package net.yacy.kelondro.workflow;++/**+ * A busy thread to run a job only once+ */+public abstract class OneTimeBusyThread extends InstantBusyThread {++ /**+ * Construct an instance able to run a job once, after a given delay+ * @param jobName the job name used to monitor the thread+ * @param startupDelay the delay in milliseconds to wait before starting the job+ */+ public OneTimeBusyThread(final String jobName, final long startupDelay) {+ super(jobName, Long.MIN_VALUE, Long.MIN_VALUE);+ this.setStartupSleep(startupDelay);+ this.setIdleSleep(-1);+ this.setBusySleep(-1);+ this.setMemPreReqisite(0);+ this.setLoadPreReqisite(+ Double.MAX_VALUE); /*+ * this is called during initialization phase and some code parts depend on it;+ * therefore we cannot set a prerequisite that prevents the start of that thread+ */+ }++ /**+ * Construct an instance able to run a job once and immediately+ * @param jobName the job name used to monitor the thread+ */+ public OneTimeBusyThread(final String jobName) {+ this(jobName, 0);+ }++ /**+ * Construct and start a OneTimeBusyThread instance from a runnable task.+ * @param task the task to run once+ * @param startupDelay the delay in milliseconds to wait before starting the job+ * @return a OneTimeBusyThread instance+ * @throws IllegalArgumentException when the task is null+ */+ public static OneTimeBusyThread startFromRunnable(final Runnable task, final long startupDelay) {+ if(task == null) {+ throw new IllegalArgumentException("Runnable task must not be null");+ }+ OneTimeBusyThread busyThread = new OneTimeBusyThread(task.getClass().getName() + ".run", startupDelay) {++ @Override+ public boolean jobImpl() throws Exception {+ task.run();+ return true;+ }++ };+ busyThread.start();+ return busyThread;+ }++}diff --git a/source/net/yacy/peers/OnePeerPingBusyThread.java b/source/net/yacy/peers/OnePeerPingBusyThread.javanew file mode 100644index 000000000..347feb638--- /dev/null+++ b/source/net/yacy/peers/OnePeerPingBusyThread.java@@ -0,0 +1,53 @@+// OnePeerPingBusyThread.java+// ---------------------------+// Copyright 2017 by luccioman; https://github.com/luccioman+//+// This is a part of YaCy, a peer-to-peer based web search engine+//+// LICENSE+//+// This program is free software; you can redistribute it and/or modify+// it under the terms of the GNU General Public License as published by+// the Free Software Foundation; either version 2 of the License, or+// (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU General Public License for more details.+//+// You should have received a copy of the GNU General Public License+// along with this program; if not, write to the Free Software+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA++package net.yacy.peers;++import net.yacy.kelondro.workflow.OneTimeBusyThread;++/**+ * A busy thread running peer ping only once.+ */+public class OnePeerPingBusyThread extends OneTimeBusyThread {++ /** Access to the peer to peer network */+ final net.yacy.peers.Network peersNetwork;++ /**+ * @param peersNetwork a net.yacy.peers.Network instance+ * @throws IllegalArgumentException when peersNetwork is null+ */+ public OnePeerPingBusyThread(final net.yacy.peers.Network peersNetwork) {+ super("Network.peerPing");+ if(peersNetwork == null) {+ throw new IllegalArgumentException("peersNetwork parameter must not be null");+ }+ this.peersNetwork = peersNetwork;+ }++ @Override+ public boolean jobImpl() throws Exception {+ this.peersNetwork.peerPing();+ return true;+ }++}\ No newline at end of filediff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.javaindex cd7e4850e..3406071f2 100644--- a/source/net/yacy/search/Switchboard.java+++ b/source/net/yacy/search/Switchboard.java@@ -193,6 +193,7 @@ import net.yacy.kelondro.util.OS;import net.yacy.kelondro.util.SetTools;import net.yacy.kelondro.workflow.BusyThread;import net.yacy.kelondro.workflow.InstantBusyThread;+import net.yacy.kelondro.workflow.OneTimeBusyThread;import net.yacy.kelondro.workflow.WorkflowProcessor;import net.yacy.kelondro.workflow.WorkflowThread;import net.yacy.peers.DHTSelection;@@ -337,7 +338,14 @@ public final class Switchboard extends serverSwitch {// UPnP port mappingif ( getConfigBool(SwitchboardConstants.UPNP_ENABLED, false) ) {- InstantBusyThread.oneTimeJob(UPnP.class, "addPortMappings", 0);+ new OneTimeBusyThread("UPnP.addPortMappings") {++ @Override+ public boolean jobImpl() throws Exception {+ UPnP.addPortMappings();+ return true;+ }+ }.start();}// init TrayIcon if possible@@ -611,7 +619,14 @@ public final class Switchboard extends serverSwitch {// start yacy corethis.log.config("Starting YaCy Protocol Core");this.yc = new Network(this);- InstantBusyThread.oneTimeJob(this, "loadSeedLists", 0);+ new OneTimeBusyThread("Switchboard.loadSeedLists") {++ @Override+ public boolean jobImpl() throws Exception {+ loadSeedLists();+ return true;+ }+ }.start();//final long startedSeedListAquisition = System.currentTimeMillis();// init a DHT transmission dispatcher@@ -941,7 +956,17 @@ public final class Switchboard extends serverSwitch {// initializing the resourceObserverthis.observer = new ResourceObserver(this);- InstantBusyThread.oneTimeJob(this.observer, "resourceObserverJob", 0);++ final ResourceObserver resourceObserver = this.observer;+ new OneTimeBusyThread("ResourceObserver.resourceObserverJob") {++ @Override+ public boolean jobImpl() throws Exception {+ resourceObserver.resourceObserverJob();+ return true;+ }+ }.start();+// initializing the stackCrawlThreadthis.crawlStacker =@@ -1032,101 +1057,142 @@ public final class Switchboard extends serverSwitch {// deploy busy threadsthis.log.config("Starting Threads");MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq-+deployThread(SwitchboardConstants.CLEANUP,"Cleanup","cleaning process",null,- new InstantBusyThread(- this,- SwitchboardConstants.CLEANUP_METHOD_START,- SwitchboardConstants.CLEANUP_METHOD_JOBCOUNT,- SwitchboardConstants.CLEANUP_METHOD_FREEMEM,- 30000,- 10000),+ new InstantBusyThread("Switchboard.cleanupJob", 30000, 10000) {++ @Override+ public boolean jobImpl() throws Exception {+ return cleanupJob();+ }++ @Override+ public int getJobCount() {+ return cleanupJobSize();+ }++ @Override+ public void freememImpl() {+ }++ },60000); // all 10 minutes, wait 1 minute until first run+deployThread(SwitchboardConstants.SCHEDULER,"Scheduler","starts scheduled processes from the API Processing table",null,- new InstantBusyThread(- this,- SwitchboardConstants.SCHEDULER_METHOD_START,- SwitchboardConstants.SCHEDULER_METHOD_JOBCOUNT,- SwitchboardConstants.SCHEDULER_METHOD_FREEMEM,- 30000,- 10000),+ new InstantBusyThread("Switchboard.schedulerJob", 30000, 10000) {+ @Override+ public boolean jobImpl() throws Exception {+ return schedulerJob();+ }++ @Override+ public int getJobCount() {+ return schedulerJobSize();+ }++ @Override+ public void freememImpl() {+ }+ },60000); // all 10 minutes, wait 1 minute until first run+deployThread(SwitchboardConstants.SURROGATES,"Surrogates","A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.",null,- new InstantBusyThread(- this,- SwitchboardConstants.SURROGATES_METHOD_START,- SwitchboardConstants.SURROGATES_METHOD_JOBCOUNT,- SwitchboardConstants.SURROGATES_METHOD_FREEMEM,- 20000,- 0),+ new InstantBusyThread("Switchboard.surrogateProcess", 20000, 0) {+ @Override+ public boolean jobImpl() throws Exception {+ return surrogateProcess();+ }++ @Override+ public int getJobCount() {+ return surrogateQueueSize();+ }++ @Override+ public void freememImpl() {+ surrogateFreeMem();+ }+ },10000);this.initRemoteCrawler(this.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));this.initAutocrawl(this.getConfigBool(SwitchboardConstants.AUTOCRAWL, false));+ final CrawlQueues crawlQueue = this.crawlQueues;deployThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL,"Local Crawl","thread that performes a single crawl step from the local crawl queue","/IndexCreateQueues_p.html?stack=LOCAL",- new InstantBusyThread(- this.crawlQueues,- SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START,- SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT,- SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM,- 0,- 0),+ new InstantBusyThread("CrawlQueues.coreCrawlJob", 0, 0) {+ @Override+ public boolean jobImpl() throws Exception {+ return crawlQueue.coreCrawlJob();+ }++ @Override+ public int getJobCount() {+ return crawlQueue.coreCrawlJobSize();+ }++ @Override+ public void freememImpl() {+ crawlQueue.freemem();+ }+ },10000);++ final Network net = this.yc;deployThread(SwitchboardConstants.SEED_UPLOAD,"Seed-List Upload","task that a principal peer performes to generate and upload a seed-list to a ftp account",null,- new InstantBusyThread(- this.yc,- SwitchboardConstants.SEED_UPLOAD_METHOD_START,- SwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT,- SwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM,- 600000,- 300000),+ new InstantBusyThread("Network.publishSeedList", 600000, 300000) {+ @Override+ public boolean jobImpl() throws Exception {+ net.publishSeedList();+ return true;+ }+ },180000);+deployThread(SwitchboardConstants.PEER_PING,"YaCy Core","this is the p2p-control and peer-ping task",null,- new InstantBusyThread(- this.yc,- SwitchboardConstants.PEER_PING_METHOD_START,- SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT,- SwitchboardConstants.PEER_PING_METHOD_FREEMEM,- 30000,- 30000),+ new InstantBusyThread("Network.peerPing", 30000, 30000) {+ @Override+ public boolean jobImpl() throws Exception {+ net.peerPing();+ return true;+ }+ },10000);deployThread(SwitchboardConstants.INDEX_DIST,"DHT Distribution","selection, transfer and deletion of index entries that are not searched on your peer, but on others",null,- new InstantBusyThread(- this,- SwitchboardConstants.INDEX_DIST_METHOD_START,- SwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT,- SwitchboardConstants.INDEX_DIST_METHOD_FREEMEM,- 10000,- 1000),+ new InstantBusyThread("Switchboard.dhtTransferJob", 10000, 1000) {+ @Override+ public boolean jobImpl() throws Exception {+ return dhtTransferJob();+ }+ },60000,Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP, "5000")),Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP, "0")),@@ -1138,28 +1204,20 @@ public final class Switchboard extends serverSwitch {"720_ccimport","Content Control Import","this is the content control import thread",- null,- new InstantBusyThread(- new SMWListSyncThread(this, sb.getConfig("contentcontrol.bookmarklist", "contentcontrol"), "Category:Content Source", "/?Url/?Filter/?Category/?Modification date", sb.getConfigBool(- "contentcontrol.smwimport.purgelistoninit", false)),- "run",- SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT,- SwitchboardConstants.PEER_PING_METHOD_FREEMEM,- 3000,- 3000),+ null,+ InstantBusyThread.createFromRunnable(+ new SMWListSyncThread(this, sb.getConfig("contentcontrol.bookmarklist", "contentcontrol"),+ "Category:Content Source", "/?Url/?Filter/?Category/?Modification date",+ sb.getConfigBool("contentcontrol.smwimport.purgelistoninit", false)),+ 3000, 3000),2000);+deployThread("730_ccfilter","Content Control Filter","this is the content control filter update thread",null,- new InstantBusyThread(- new ContentControlFilterUpdateThread(this),- "run",- SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT,- SwitchboardConstants.PEER_PING_METHOD_FREEMEM,- 3000,- 3000),+ InstantBusyThread.createFromRunnable(new ContentControlFilterUpdateThread(this), 3000, 3000),2000);// set network-specific performance attributes@@ -1513,6 +1571,8 @@ public final class Switchboard extends serverSwitch {this.peers.mySeed().setFlagAcceptRemoteCrawl(activate);if (activate) {this.crawlQueues.initRemoteCrawlQueues();++ final CrawlQueues queues = this.crawlQueues;BusyThread rct = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);if (rct == null) {@@ -1521,13 +1581,19 @@ public final class Switchboard extends serverSwitch {"Remote Crawl Job","thread that performes a single crawl/indexing step triggered by a remote peer","/IndexCreateQueues_p.html?stack=REMOTE",- new InstantBusyThread(- this.crawlQueues,- SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START,- SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT,- SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM,- 0,- 0),+ new InstantBusyThread("CrawlQueues.remoteTriggeredCrawlJob", 0, 0) {++ @Override+ public boolean jobImpl() throws Exception {+ return queues.remoteTriggeredCrawlJob();+ }++ @Override+ public int getJobCount() {+ return queues.remoteTriggeredCrawlJobSize();+ }++ },10000);rct = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);}@@ -1541,13 +1607,12 @@ public final class Switchboard extends serverSwitch {"Remote Crawl URL Loader","thread that loads remote crawl lists from other peers",null,- new InstantBusyThread(- this.crawlQueues,- SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START,- SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT,- SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM,- 10000,- 10000),+ new InstantBusyThread("CrawlQueues.remoteCrawlLoaderJob", 10000, 10000) {+ @Override+ public boolean jobImpl() throws Exception {+ return queues.remoteCrawlLoaderJob();+ }+ },10000);rcl = getThread(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER);@@ -1569,18 +1634,19 @@ public final class Switchboard extends serverSwitch {if (activate) {BusyThread acr = getThread(SwitchboardConstants.CRAWLJOB_AUTOCRAWL);if (acr == null) {+ final CrawlQueues queues = this.crawlQueues;+deployThread(SwitchboardConstants.CRAWLJOB_AUTOCRAWL,"Autocrawl","Thread that selects and automatically adds crawling jobs to the local queue",null,- new InstantBusyThread(- this.crawlQueues,- SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_START,- SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_JOBCOUNT,- SwitchboardConstants.CRAWLJOB_AUTOCRAWL_METHOD_FREEMEM,- 10000,- 10000),+ new InstantBusyThread("CrawlQueues.autocrawlJob", 10000, 10000) {+ @Override+ public boolean jobImpl() throws Exception {+ return queues.autocrawlJob();+ }+ },10000);acr = getThread(SwitchboardConstants.CRAWLJOB_AUTOCRAWL);@@ -2832,6 +2898,11 @@ public final class Switchboard extends serverSwitch {}}+ /**+ * Parse a response to produce a new document to add to the index.+ * <strong>Important :</strong> this method is called using reflection as a Workflow process and must therefore remain public.+ * @param in an indexing workflow entry containing a response to parse+ */public IndexingQueueEntry parseDocument(final IndexingQueueEntry in) {in.queueEntry.updateStatus(Response.QUEUE_STATE_PARSING);Document[] documents = null;@@ -3009,6 +3080,10 @@ public final class Switchboard extends serverSwitch {return documents;}+ /**+ * <strong>Important :</strong> this method is called using reflection as a Workflow process and must therefore remain public.+ * @param in an indexing workflow entry containing a response and the related parsed document(s)+ */public IndexingQueueEntry condenseDocument(final IndexingQueueEntry in) {in.queueEntry.updateStatus(Response.QUEUE_STATE_CONDENSING);CrawlProfile profile = in.queueEntry.profile();@@ -3083,6 +3158,11 @@ public final class Switchboard extends serverSwitch {return new IndexingQueueEntry(in.queueEntry, in.documents, condenser);}+ /**+ * Perform web structure analysis on parsed documents and update the web structure graph.+ * <strong>Important :</strong> this method is called using reflection as a Workflow process and must therefore remain public.+ * @param in an indexing workflow entry containing parsed document(s)+ */public IndexingQueueEntry webStructureAnalysis(final IndexingQueueEntry in) {in.queueEntry.updateStatus(Response.QUEUE_STATE_STRUCTUREANALYSIS);for (Document document : in.documents) {@@ -3096,6 +3176,11 @@ public final class Switchboard extends serverSwitch {return in;}+ /**+ * Store a new entry to the local index.+ * <strong>Important :</strong> this method is called using reflection as a Workflow process and must therefore remain public.+ * @param in an indexing workflow entry containing parsed document(s) and a condenser instance+ */public void storeDocumentIndex(final IndexingQueueEntry in) {in.queueEntry.updateStatus(Response.QUEUE_STATE_INDEXSTORAGE);// the condenser may be null in case that an indexing is not wanted (there may be a no-indexing flag in the file)diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.javaindex af90841bd..013796b74 100644--- a/source/net/yacy/search/SwitchboardConstants.java+++ b/source/net/yacy/search/SwitchboardConstants.java@@ -87,9 +87,6 @@ public final class SwitchboardConstants {* connected to the YaCy-network</p>*/public static final String PEER_PING = "30_peerping";- public static final String PEER_PING_METHOD_START = "peerPing";- public static final String PEER_PING_METHOD_JOBCOUNT = null;- public static final String PEER_PING_METHOD_FREEMEM = null;public static final String PEER_PING_IDLESLEEP = "30_peerping_idlesleep";public static final String PEER_PING_BUSYSLEEP = "30_peerping_busysleep";// 40_peerseedcycle@@ -98,9 +95,6 @@ public final class SwitchboardConstants {* <p>Name of the seed upload thread, providing the so-called seed-lists needed during bootstrapping</p>*/public static final String SEED_UPLOAD = "40_peerseedcycle";- public static final String SEED_UPLOAD_METHOD_START = "publishSeedList";- public static final String SEED_UPLOAD_METHOD_JOBCOUNT = null;- public static final String SEED_UPLOAD_METHOD_FREEMEM = null;public static final String SEED_UPLOAD_IDLESLEEP = "40_peerseedcycle_idlesleep";public static final String SEED_UPLOAD_BUSYSLEEP = "40_peerseedcycle_busysleep";// 50_localcrawl@@ -111,9 +105,6 @@ public final class SwitchboardConstants {**/public static final String CRAWLJOB_LOCAL_CRAWL = "50_localcrawl";- public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_START = "coreCrawlJob";- public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT = "coreCrawlJobSize";- public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM = "freemem";public static final String CRAWLJOB_LOCAL_CRAWL_IDLESLEEP = "50_localcrawl_idlesleep";public static final String CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP = "50_localcrawl_busysleep";public static final String CRAWLJOB_LOCAL_CRAWL_LOADPREREQ = "50_localcrawl_loadprereq";@@ -148,9 +139,6 @@ public final class SwitchboardConstants {* <p>Name of the remote triggered crawl thread, responsible for processing a remote crawl received from another peer</p>*/public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL = "62_remotetriggeredcrawl";- public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START = "remoteTriggeredCrawlJob";- public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT = "remoteTriggeredCrawlJobSize";- public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM = null;public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP = "62_remotetriggeredcrawl_idlesleep";public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP = "62_remotetriggeredcrawl_busysleep";// 70_surrogates@@ -163,18 +151,12 @@ public final class SwitchboardConstants {public static final String SURROGATES_LOADPREREQ = "70_surrogates_loadprereq";public static final String SURROGATES_IDLESLEEP = "70_surrogates_idlesleep";public static final String SURROGATES_BUSYSLEEP = "70_surrogates_busysleep";- public static final String SURROGATES_METHOD_START = "surrogateProcess";- public static final String SURROGATES_METHOD_JOBCOUNT = "surrogateQueueSize";- public static final String SURROGATES_METHOD_FREEMEM = "surrogateFreeMem";// 85_scheduler/*** <p><code>public static final String <strong>SCHEDULER</strong> = "85_scheduler"</code></p>* <p>The cleanup thread which is responsible for the start of scheduled processes from the API table</p>*/public static final String SCHEDULER = "85_scheduler";- public static final String SCHEDULER_METHOD_START = "schedulerJob";- public static final String SCHEDULER_METHOD_JOBCOUNT = "schedulerJobSize";- public static final String SCHEDULER_METHOD_FREEMEM = null;public static final String SCHEDULER_IDLESLEEP = "85_scheduler_idlesleep";public static final String SCHEDULER_BUSYSLEEP = "85_scheduler_busysleep";// 90_cleanup@@ -183,9 +165,6 @@ public final class SwitchboardConstants {* <p>The cleanup thread which is responsible for pendant cleanup-jobs, news/ranking distribution, etc.</p>*/public static final String CLEANUP = "90_cleanup";- public static final String CLEANUP_METHOD_START = "cleanupJob";- public static final String CLEANUP_METHOD_JOBCOUNT = "cleanupJobSize";- public static final String CLEANUP_METHOD_FREEMEM = null;public static final String CLEANUP_IDLESLEEP = "90_cleanup_idlesleep";public static final String CLEANUP_BUSYSLEEP = "90_cleanup_busysleep";/**diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.javaindex c56fd2b89..c2b44aa5b 100644--- a/source/net/yacy/search/query/SearchEvent.java+++ b/source/net/yacy/search/query/SearchEvent.java@@ -521,6 +521,9 @@ public final class SearchEvent {SearchEventCache.put(this.query.id(false), this);}+ /**+ * A concurrent task to perform the current search query on the local RWI.+ */private class RWIProcess extends Thread {final Thread waitForThread;@@ -530,6 +533,9 @@ public final class SearchEvent {this.waitForThread = waitForThread;}+ /**+ * Query the local RWI and feed the search event with the obtained results.+ */@Overridepublic void run() {