Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
creating a threaddump during every cleanup process to be able to find out what a peer did (not) last time before a crash
Michael Peter Christen22841ffbf11d
diff --git a/htroot/Threaddump_p.java b/htroot/Threaddump_p.javaindex 32bd879f0..de52b3fd9 100644--- a/htroot/Threaddump_p.java+++ b/htroot/Threaddump_p.java@@ -25,20 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA--import java.io.File;-import java.io.IOException;-import java.lang.management.ManagementFactory;-import java.lang.management.ThreadInfo;-import java.lang.management.ThreadMXBean;-import java.util.ArrayList;-import java.util.Date;-import java.util.Map;-import net.yacy.cora.protocol.RequestHeader;import net.yacy.kelondro.logging.ThreadDump;-import net.yacy.kelondro.util.OS;-import net.yacy.peers.operation.yacyBuildProperties;import net.yacy.search.Switchboard;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;@@ -47,84 +35,26 @@ public class Threaddump_p {public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {- serverObjects prop = new serverObjects();- Switchboard sb = (Switchboard) env;-- final StringBuilder buffer = new StringBuilder(1000);-- final boolean plain = post != null && post.getBoolean("plain");- final int sleep = (post == null) ? 0 : post.getInt("sleep", 0); // a sleep before creation of a thread dump can be used for profiling- if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {}- prop.put("dump", "1");- // Thread dump- final Date dt = new Date();- final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();- Runtime runtime = Runtime.getRuntime();-- ThreadDump.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************");- ThreadDump.bufferappend(buffer, plain, " ");- ThreadDump.bufferappend(buffer, plain, "YaCy Version: " + versionstring);- ThreadDump.bufferappend(buffer, plain, "Assigned Memory = " + (runtime.maxMemory()));- ThreadDump.bufferappend(buffer, plain, "Used Memory = " + (runtime.totalMemory() - runtime.freeMemory()));- ThreadDump.bufferappend(buffer, plain, "Available Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()));- ThreadDump.bufferappend(buffer, plain, " ");- ThreadDump.bufferappend(buffer, plain, " ");+ serverObjects prop = new serverObjects();+ Switchboard sb = (Switchboard) env;- int multipleCount = 100;- File appPath = sb.getAppPath();- if (post != null && post.containsKey("multipleThreaddump")) {- multipleCount = post.getInt("count", multipleCount);- final ArrayList<Map<Thread,StackTraceElement[]>> traces = new ArrayList<Map<Thread,StackTraceElement[]>>();- for (int i = 0; i < multipleCount; i++) {- try {- traces.add(ThreadDump.getAllStackTraces());- } catch (final OutOfMemoryError e) {- break;- }- }- ThreadDump.appendStackTraceStats(appPath, buffer, traces, plain);- } else {- // write a thread dump to standard error output- File logFile = new File("yacy.log");- if (ThreadDump.canProduceLockedBy(logFile)) {- try {- new ThreadDump(logFile).appendBlockTraces(buffer, plain);- } catch (final IOException e) {- e.printStackTrace();- }- } else if (OS.canExecUnix) {- ThreadDump.bufferappend(buffer, plain, "this thread dump function can find threads that lock others, to enable this function start YaCy with 'startYACY.sh -l'");- ThreadDump.bufferappend(buffer, plain, " ");- }+ final boolean plain = post != null && post.getBoolean("plain");+ final int sleep = (post == null) ? 0 : post.getInt("sleep", 0); // a sleep before creation of a thread dump can be used for profiling+ if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {}+ prop.put("dump", "1");- // generate a single thread dump- final Map<Thread,StackTraceElement[]> stackTraces = ThreadDump.getAllStackTraces();- new ThreadDump(appPath, stackTraces, plain, Thread.State.BLOCKED).appendStackTraces(buffer, plain, Thread.State.BLOCKED);- new ThreadDump(appPath, stackTraces, plain, Thread.State.RUNNABLE).appendStackTraces(buffer, plain, Thread.State.RUNNABLE);- new ThreadDump(appPath, stackTraces, plain, Thread.State.TIMED_WAITING).appendStackTraces(buffer, plain, Thread.State.TIMED_WAITING);- new ThreadDump(appPath, stackTraces, plain, Thread.State.WAITING).appendStackTraces(buffer, plain, Thread.State.WAITING);- new ThreadDump(appPath, stackTraces, plain, Thread.State.NEW).appendStackTraces(buffer, plain, Thread.State.NEW);- new ThreadDump(appPath, stackTraces, plain, Thread.State.TERMINATED).appendStackTraces(buffer, plain, Thread.State.TERMINATED);+ int multipleCount = 100;+ boolean multiple = post != null && post.containsKey("multipleThreaddump");+ if (multiple) {+ multipleCount = post.getInt("count", multipleCount);}- ThreadDump.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************");+ String threaddump = ThreadDump.threaddump(sb, plain, sleep, multiple, multipleCount);+ prop.put("plain_count", multipleCount);+ prop.put("plain_content", threaddump);+ prop.put("plain", (plain) ? 1 : 0);-- ThreadDump.bufferappend(buffer, plain, "");- ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();- ThreadDump.bufferappend(buffer, plain, "Thread list from ThreadMXBean, " + threadbean.getThreadCount() + " threads:");- ThreadInfo[] threadinfo = threadbean.dumpAllThreads(true, true);- for (ThreadInfo ti: threadinfo) {- ThreadDump.bufferappend(buffer, plain, ti.getThreadName());- }-- prop.put("plain_count", multipleCount);- prop.put("plain_content", buffer.toString());- prop.put("plain", (plain) ? 1 : 0);-- return prop; // return from serverObjects respond()+ return prop;}--}diff --git a/source/net/yacy/kelondro/logging/ThreadDump.java b/source/net/yacy/kelondro/logging/ThreadDump.javaindex c554d304c..9f1f24eaa 100644--- a/source/net/yacy/kelondro/logging/ThreadDump.java+++ b/source/net/yacy/kelondro/logging/ThreadDump.java@@ -32,7 +32,11 @@ import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.RandomAccessFile;+import java.lang.management.ManagementFactory;+import java.lang.management.ThreadInfo;+import java.lang.management.ThreadMXBean;import java.util.ArrayList;+import java.util.Date;import java.util.HashMap;import java.util.List;import java.util.Map;@@ -41,6 +45,8 @@ import java.util.regex.Pattern;import net.yacy.document.parser.html.CharacterCoding;import net.yacy.kelondro.util.FileUtils;import net.yacy.kelondro.util.OS;+import net.yacy.peers.operation.yacyBuildProperties;+import net.yacy.search.Switchboard;import net.yacy.utils.nxTools;public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> implements Map<ThreadDump.StackTrace, List<String>> {@@ -372,7 +378,7 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp}public static void bufferappend(final StringBuilder buffer, final boolean plain, final String a) {- buffer.append(a);+ buffer.append(plain ? a.replaceAll(" ", "") : a);buffer.append(plain ? "\n" : "<br />");}@@ -449,6 +455,73 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> impSystem.out.println("");}+ public static String threaddump(Switchboard sb, boolean plain, int sleep, boolean multiple, int multipleCount) {+ final StringBuilder buffer = new StringBuilder(1000);++ if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {}+ // Thread dump+ final Date dt = new Date();+ final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();+ Runtime runtime = Runtime.getRuntime();++ ThreadDump.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************");+ ThreadDump.bufferappend(buffer, plain, " ");+ ThreadDump.bufferappend(buffer, plain, "YaCy Version: " + versionstring);+ ThreadDump.bufferappend(buffer, plain, "Assigned Memory = " + (runtime.maxMemory()));+ ThreadDump.bufferappend(buffer, plain, "Used Memory = " + (runtime.totalMemory() - runtime.freeMemory()));+ ThreadDump.bufferappend(buffer, plain, "Available Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()));+ ThreadDump.bufferappend(buffer, plain, " ");+ ThreadDump.bufferappend(buffer, plain, " ");++ File appPath = sb.getAppPath();+ if (multiple) {+ final ArrayList<Map<Thread,StackTraceElement[]>> traces = new ArrayList<Map<Thread,StackTraceElement[]>>();+ for (int i = 0; i < multipleCount; i++) {+ try {+ traces.add(ThreadDump.getAllStackTraces());+ } catch (final OutOfMemoryError e) {+ break;+ }+ }+ ThreadDump.appendStackTraceStats(appPath, buffer, traces, plain);+ } else {+ // write a thread dump to standard error output+ File logFile = new File("yacy.log");+ if (ThreadDump.canProduceLockedBy(logFile)) {+ try {+ new ThreadDump(logFile).appendBlockTraces(buffer, plain);+ } catch (final IOException e) {+ e.printStackTrace();+ }+ } else if (OS.canExecUnix) {+ ThreadDump.bufferappend(buffer, plain, "this thread dump function can find threads that lock others, to enable this function start YaCy with 'startYACY.sh -l'");+ ThreadDump.bufferappend(buffer, plain, " ");+ }++ // generate a single thread dump+ final Map<Thread,StackTraceElement[]> stackTraces = ThreadDump.getAllStackTraces();+ new ThreadDump(appPath, stackTraces, plain, Thread.State.BLOCKED).appendStackTraces(buffer, plain, Thread.State.BLOCKED);+ new ThreadDump(appPath, stackTraces, plain, Thread.State.RUNNABLE).appendStackTraces(buffer, plain, Thread.State.RUNNABLE);+ new ThreadDump(appPath, stackTraces, plain, Thread.State.TIMED_WAITING).appendStackTraces(buffer, plain, Thread.State.TIMED_WAITING);+ new ThreadDump(appPath, stackTraces, plain, Thread.State.WAITING).appendStackTraces(buffer, plain, Thread.State.WAITING);+ new ThreadDump(appPath, stackTraces, plain, Thread.State.NEW).appendStackTraces(buffer, plain, Thread.State.NEW);+ new ThreadDump(appPath, stackTraces, plain, Thread.State.TERMINATED).appendStackTraces(buffer, plain, Thread.State.TERMINATED);+ }++ ThreadDump.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************");++ ThreadDump.bufferappend(buffer, plain, "");+ ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();+ ThreadDump.bufferappend(buffer, plain, "Thread list from ThreadMXBean, " + threadbean.getThreadCount() + " threads:");+ ThreadInfo[] threadinfo = threadbean.dumpAllThreads(true, true);+ for (ThreadInfo ti: threadinfo) {+ ThreadDump.bufferappend(buffer, plain, ti.getThreadName());+ }++ return buffer.toString();+ }++public static void main(final String[] args) {ThreadDump dump = null;if (args.length == 2 && args[0].equals("-f")) {diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.javaindex 6d823344b..fb6178ab0 100644--- a/source/net/yacy/search/Switchboard.java+++ b/source/net/yacy/search/Switchboard.java@@ -49,6 +49,7 @@ import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.OutputStream;+import java.io.PrintWriter;import java.io.Reader;import java.net.InetAddress;import java.net.MalformedURLException;@@ -193,6 +194,7 @@ import net.yacy.kelondro.blob.Tables.SortDirection;import net.yacy.kelondro.data.meta.URIMetadataNode;import net.yacy.kelondro.data.word.Word;import net.yacy.kelondro.logging.GuiHandler;+import net.yacy.kelondro.logging.ThreadDump;import net.yacy.kelondro.rwi.ReferenceContainer;import net.yacy.kelondro.util.FileUtils;import net.yacy.kelondro.util.MemoryControl;@@ -2593,11 +2595,22 @@ public final class Switchboard extends serverSwitch {}public boolean cleanupJob() {-+ConcurrentLog.ensureWorkerIsRunning();try {clearCaches();+ // write a thread dump to log path+ try {+ File tdlog = new File(dataPath, "DATA/LOG/threaddump.txt");+ PrintWriter out = new PrintWriter(tdlog);+ String threaddump = ThreadDump.threaddump(this, true, 0, false, 0);+ out.println(threaddump);+ out.close();+ } catch (IOException e) {+ log.info("cannot write threaddump", e);+ }+// clear caches if necessaryif ( !MemoryControl.request(128000000L, false) ) {this.index.clearCaches();@@ -2621,7 +2634,7 @@ public final class Switchboard extends serverSwitch {log.info("finishing greedy learning phase, size=" +cs);}}-+// refresh recrawl datestry {CrawlProfile selentry;