Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

creating a threaddump during every cleanup process to be able to find out what a peer did (not) last time before a crash

Michael Peter Christen22841ffbf11d
diff --git a/htroot/Threaddump_p.java b/htroot/Threaddump_p.java
index 32bd879f0..de52b3fd9 100644
--- a/htroot/Threaddump_p.java
+++ b/htroot/Threaddump_p.java
@@ -25,20 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadInfo;
-import java.lang.management.ThreadMXBean;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.Map;
-
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.ThreadDump;
-import net.yacy.kelondro.util.OS;
-import net.yacy.peers.operation.yacyBuildProperties;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@@ -47,84 +35,26 @@ public class Threaddump_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
- serverObjects prop = new serverObjects();
- Switchboard sb = (Switchboard) env;
-
- final StringBuilder buffer = new StringBuilder(1000);
-
- final boolean plain = post != null && post.getBoolean("plain");
- final int sleep = (post == null) ? 0 : post.getInt("sleep", 0); // a sleep before creation of a thread dump can be used for profiling
- if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {}
- prop.put("dump", "1");
- // Thread dump
- final Date dt = new Date();
- final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
- Runtime runtime = Runtime.getRuntime();
-
- ThreadDump.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************");
- ThreadDump.bufferappend(buffer, plain, " ");
- ThreadDump.bufferappend(buffer, plain, "YaCy Version: " + versionstring);
- ThreadDump.bufferappend(buffer, plain, "Assigned   Memory = " + (runtime.maxMemory()));
- ThreadDump.bufferappend(buffer, plain, "Used       Memory = " + (runtime.totalMemory() - runtime.freeMemory()));
- ThreadDump.bufferappend(buffer, plain, "Available  Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()));
- ThreadDump.bufferappend(buffer, plain, " ");
- ThreadDump.bufferappend(buffer, plain, " ");
+ serverObjects prop = new serverObjects();
+ Switchboard sb = (Switchboard) env;
- int multipleCount = 100;
- File appPath = sb.getAppPath();
- if (post != null && post.containsKey("multipleThreaddump")) {
- multipleCount = post.getInt("count", multipleCount);
- final ArrayList<Map<Thread,StackTraceElement[]>> traces = new ArrayList<Map<Thread,StackTraceElement[]>>();
- for (int i = 0; i < multipleCount; i++) {
- try {
- traces.add(ThreadDump.getAllStackTraces());
- } catch (final OutOfMemoryError e) {
- break;
- }
- }
- ThreadDump.appendStackTraceStats(appPath, buffer, traces, plain);
- } else {
- // write a thread dump to standard error output
- File logFile = new File("yacy.log");
- if (ThreadDump.canProduceLockedBy(logFile)) {
- try {
- new ThreadDump(logFile).appendBlockTraces(buffer, plain);
- } catch (final IOException e) {
- e.printStackTrace();
- }
- } else if (OS.canExecUnix) {
- ThreadDump.bufferappend(buffer, plain, "this thread dump function can find threads that lock others, to enable this function start YaCy with 'startYACY.sh -l'");
- ThreadDump.bufferappend(buffer, plain, "&nbsp;");
- }
+ final boolean plain = post != null && post.getBoolean("plain");
+ final int sleep = (post == null) ? 0 : post.getInt("sleep", 0); // a sleep before creation of a thread dump can be used for profiling
+ if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {}
+ prop.put("dump", "1");
- // generate a single thread dump
- final Map<Thread,StackTraceElement[]> stackTraces = ThreadDump.getAllStackTraces();
- new ThreadDump(appPath, stackTraces, plain, Thread.State.BLOCKED).appendStackTraces(buffer, plain, Thread.State.BLOCKED);
- new ThreadDump(appPath, stackTraces, plain, Thread.State.RUNNABLE).appendStackTraces(buffer, plain, Thread.State.RUNNABLE);
- new ThreadDump(appPath, stackTraces, plain, Thread.State.TIMED_WAITING).appendStackTraces(buffer, plain, Thread.State.TIMED_WAITING);
- new ThreadDump(appPath, stackTraces, plain, Thread.State.WAITING).appendStackTraces(buffer, plain, Thread.State.WAITING);
- new ThreadDump(appPath, stackTraces, plain, Thread.State.NEW).appendStackTraces(buffer, plain, Thread.State.NEW);
- new ThreadDump(appPath, stackTraces, plain, Thread.State.TERMINATED).appendStackTraces(buffer, plain, Thread.State.TERMINATED);
+ int multipleCount = 100;
+ boolean multiple = post != null && post.containsKey("multipleThreaddump");
+ if (multiple) {
+ multipleCount = post.getInt("count", multipleCount);
}
- ThreadDump.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************");
+ String threaddump = ThreadDump.threaddump(sb, plain, sleep, multiple, multipleCount);
+ prop.put("plain_count", multipleCount);
+ prop.put("plain_content", threaddump);
+ prop.put("plain", (plain) ? 1 : 0);
-
- ThreadDump.bufferappend(buffer, plain, "");
- ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();
- ThreadDump.bufferappend(buffer, plain, "Thread list from ThreadMXBean, " + threadbean.getThreadCount() + " threads:");
- ThreadInfo[] threadinfo = threadbean.dumpAllThreads(true, true);
- for (ThreadInfo ti: threadinfo) {
- ThreadDump.bufferappend(buffer, plain, ti.getThreadName());
- }
-
- prop.put("plain_count", multipleCount);
- prop.put("plain_content", buffer.toString());
- prop.put("plain", (plain) ? 1 : 0);
-
- return prop; // return from serverObjects respond()
+ return prop;
}
-
-
}
diff --git a/source/net/yacy/kelondro/logging/ThreadDump.java b/source/net/yacy/kelondro/logging/ThreadDump.java
index c554d304c..9f1f24eaa 100644
--- a/source/net/yacy/kelondro/logging/ThreadDump.java
+++ b/source/net/yacy/kelondro/logging/ThreadDump.java
@@ -32,7 +32,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -41,6 +45,8 @@ import java.util.regex.Pattern;
import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.OS;
+import net.yacy.peers.operation.yacyBuildProperties;
+import net.yacy.search.Switchboard;
import net.yacy.utils.nxTools;
public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> implements Map<ThreadDump.StackTrace, List<String>> {
@@ -372,7 +378,7 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
}
public static void bufferappend(final StringBuilder buffer, final boolean plain, final String a) {
- buffer.append(a);
+ buffer.append(plain ? a.replaceAll("&nbsp;", "") : a);
buffer.append(plain ? "\n" : "<br />");
}
@@ -449,6 +455,73 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
System.out.println("");
}
+ public static String threaddump(Switchboard sb, boolean plain, int sleep, boolean multiple, int multipleCount) {
+ final StringBuilder buffer = new StringBuilder(1000);
+
+ if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {}
+ // Thread dump
+ final Date dt = new Date();
+ final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
+ Runtime runtime = Runtime.getRuntime();
+
+ ThreadDump.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************");
+ ThreadDump.bufferappend(buffer, plain, "&nbsp;");
+ ThreadDump.bufferappend(buffer, plain, "YaCy Version: " + versionstring);
+ ThreadDump.bufferappend(buffer, plain, "Assigned&nbsp;&nbsp;&nbsp;Memory = " + (runtime.maxMemory()));
+ ThreadDump.bufferappend(buffer, plain, "Used&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Memory = " + (runtime.totalMemory() - runtime.freeMemory()));
+ ThreadDump.bufferappend(buffer, plain, "Available&nbsp;&nbsp;Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()));
+ ThreadDump.bufferappend(buffer, plain, "&nbsp;");
+ ThreadDump.bufferappend(buffer, plain, "&nbsp;");
+
+ File appPath = sb.getAppPath();
+ if (multiple) {
+ final ArrayList<Map<Thread,StackTraceElement[]>> traces = new ArrayList<Map<Thread,StackTraceElement[]>>();
+ for (int i = 0; i < multipleCount; i++) {
+ try {
+ traces.add(ThreadDump.getAllStackTraces());
+ } catch (final OutOfMemoryError e) {
+ break;
+ }
+ }
+ ThreadDump.appendStackTraceStats(appPath, buffer, traces, plain);
+ } else {
+ // write a thread dump to standard error output
+ File logFile = new File("yacy.log");
+ if (ThreadDump.canProduceLockedBy(logFile)) {
+ try {
+ new ThreadDump(logFile).appendBlockTraces(buffer, plain);
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+ } else if (OS.canExecUnix) {
+ ThreadDump.bufferappend(buffer, plain, "this thread dump function can find threads that lock others, to enable this function start YaCy with 'startYACY.sh -l'");
+ ThreadDump.bufferappend(buffer, plain, "&nbsp;");
+ }
+
+ // generate a single thread dump
+ final Map<Thread,StackTraceElement[]> stackTraces = ThreadDump.getAllStackTraces();
+ new ThreadDump(appPath, stackTraces, plain, Thread.State.BLOCKED).appendStackTraces(buffer, plain, Thread.State.BLOCKED);
+ new ThreadDump(appPath, stackTraces, plain, Thread.State.RUNNABLE).appendStackTraces(buffer, plain, Thread.State.RUNNABLE);
+ new ThreadDump(appPath, stackTraces, plain, Thread.State.TIMED_WAITING).appendStackTraces(buffer, plain, Thread.State.TIMED_WAITING);
+ new ThreadDump(appPath, stackTraces, plain, Thread.State.WAITING).appendStackTraces(buffer, plain, Thread.State.WAITING);
+ new ThreadDump(appPath, stackTraces, plain, Thread.State.NEW).appendStackTraces(buffer, plain, Thread.State.NEW);
+ new ThreadDump(appPath, stackTraces, plain, Thread.State.TERMINATED).appendStackTraces(buffer, plain, Thread.State.TERMINATED);
+ }
+
+ ThreadDump.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************");
+
+ ThreadDump.bufferappend(buffer, plain, "");
+ ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();
+ ThreadDump.bufferappend(buffer, plain, "Thread list from ThreadMXBean, " + threadbean.getThreadCount() + " threads:");
+ ThreadInfo[] threadinfo = threadbean.dumpAllThreads(true, true);
+ for (ThreadInfo ti: threadinfo) {
+ ThreadDump.bufferappend(buffer, plain, ti.getThreadName());
+ }
+
+ return buffer.toString();
+ }
+
+
public static void main(final String[] args) {
ThreadDump dump = null;
if (args.length == 2 && args[0].equals("-f")) {
diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index 6d823344b..fb6178ab0 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -49,6 +49,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
+import java.io.PrintWriter;
import java.io.Reader;
import java.net.InetAddress;
import java.net.MalformedURLException;
@@ -193,6 +194,7 @@ import net.yacy.kelondro.blob.Tables.SortDirection;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.logging.GuiHandler;
+import net.yacy.kelondro.logging.ThreadDump;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
@@ -2593,11 +2595,22 @@ public final class Switchboard extends serverSwitch {
}
public boolean cleanupJob() {
-
+
ConcurrentLog.ensureWorkerIsRunning();
try {
clearCaches();
+ // write a thread dump to log path
+ try {
+ File tdlog = new File(dataPath, "DATA/LOG/threaddump.txt");
+ PrintWriter out = new PrintWriter(tdlog);
+ String threaddump = ThreadDump.threaddump(this, true, 0, false, 0);
+ out.println(threaddump);
+ out.close();
+ } catch (IOException e) {
+ log.info("cannot write threaddump", e);
+ }
+
// clear caches if necessary
if ( !MemoryControl.request(128000000L, false) ) {
this.index.clearCaches();
@@ -2621,7 +2634,7 @@ public final class Switchboard extends serverSwitch {
log.info("finishing greedy learning phase, size=" +cs);
}
}
-
+
// refresh recrawl dates
try {
CrawlProfile selentry;