Skip to content

Lightpanda: the headless browser designed for AI and automation

Warn from the watchdog thread when a stall never returns to JavaScript

Scott Taylorbfcd5ebd280f
diff --git a/src/Watchdog.zig b/src/Watchdog.zig
index abbf3b07b..f3bfcaafd 100644
--- a/src/Watchdog.zig
+++ b/src/Watchdog.zig
@@ -26,6 +26,12 @@ const log = lp.log;
// How often the checker thread scans the entries.
const CHECK_INTERVAL_NS = 1 * std.time.ns_per_s;
+// The "watchdog stall script" line is logged by the termination interrupt,
+// which only runs once the worker is back in JavaScript. If it's still pending
+// this long after the stall was detected, the worker is stuck in native code:
+// say so from here, or the stall leaves no trace above debug.
+const NATIVE_STALL_REPORT_MS = 5_000;
+
const Watchdog = @This();
// null == disabled: no thread, register/unregister no-op.
@@ -41,6 +47,9 @@ pub const Entry = struct {
env: *Env,
heartbeat: *Heartbeat,
fired: bool = false,
+ // Set once the current stall has been reported as native (see
+ // NATIVE_STALL_REPORT_MS). Reset when a new stall fires.
+ native_reported: bool = false,
registered: bool = false,
node: std.DoublyLinkedList.Node = .{},
};
@@ -95,8 +104,6 @@ pub fn unregister(self: *Watchdog, entry: *Entry) void {
}
fn run(self: *Watchdog) void {
- const timeout_ms: u64 = self.timeout_ms.?;
-
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
@@ -105,38 +112,58 @@ fn run(self: *Watchdog) void {
if (self.shutdown) {
return;
}
+ self.scan(lp.datetime.milliTimestamp(.boot));
+ }
+}
+
+// Called with the mutex held.
+fn scan(self: *Watchdog, now: u64) void {
+ const timeout_ms: u64 = self.timeout_ms.?;
+
+ var node = self.entries.first;
+ while (node) |n| : (node = n.next) {
+ const entry: *Entry = @fieldParentPtr("node", n);
+ const heartbeat = entry.heartbeat;
+
+ if (heartbeat.wait_depth.load(.acquire) > 0) {
+ // The entry is in a controlled (e.g. non-JS) wait
+ entry.fired = false;
+ continue;
+ }
+
+ const last = heartbeat.last_activity.load(.acquire);
+ if (last == 0) {
+ // disarmed: no page work can be running
+ continue;
+ }
+
+ const stalled_ms = now -| last;
+ if (stalled_ms < timeout_ms) {
+ entry.fired = false;
+ continue;
+ }
- const now = lp.datetime.milliTimestamp(.boot);
- var node = self.entries.first;
- while (node) |n| : (node = n.next) {
- const entry: *Entry = @fieldParentPtr("node", n);
- const heartbeat = entry.heartbeat;
-
- if (heartbeat.wait_depth.load(.acquire) > 0) {
- // The entry is in a controlled (e.g. non-JS) wait
- entry.fired = false;
- continue;
- }
-
- const last = heartbeat.last_activity.load(.acquire);
- if (last == 0) {
- // disarmed: no page work can be running
- continue;
- }
-
- const stalled_ms = now -| last;
- if (stalled_ms < timeout_ms) {
- entry.fired = false;
- continue;
- }
-
- if (entry.fired == false) {
- entry.fired = true;
- log.debug(.app, "watchdog stall", .{ .stalled_ms = stalled_ms });
- // Also logs "watchdog stall script" (URL and JS stack) from
- // the worker thread when the termination interrupt lands.
- entry.env.requestTerminateForStall();
- }
+ if (entry.fired == false) {
+ entry.fired = true;
+ entry.native_reported = false;
+ log.debug(.watchdog, "watchdog stall", .{ .stalled_ms = stalled_ms });
+ // The worker logs "watchdog stall script" (URL and JS stack) when
+ // the termination interrupt lands.
+ entry.env.requestTerminateForStall(stalled_ms);
+ continue;
+ }
+
+ if (entry.native_reported) {
+ continue;
+ }
+ const requested_at = entry.env.pendingStallReport() orelse continue;
+ const pending_ms = now -| requested_at;
+ if (pending_ms >= NATIVE_STALL_REPORT_MS) {
+ entry.native_reported = true;
+ log.warn(.watchdog, "watchdog stall native", .{
+ .stalled_ms = stalled_ms,
+ .pending_ms = pending_ms,
+ });
}
}
}
@@ -177,3 +204,73 @@ pub const Heartbeat = struct {
_ = self.wait_depth.fetchSub(1, .release);
}
};
+
+const testing = @import("testing.zig");
+test "Watchdog: a stall that never returns to JavaScript is reported once" {
+ // Only the native warn: the first line is debug, and the termination
+ // interrupt never lands because no JavaScript runs.
+ testing.expectLog(&.{.watchdog});
+
+ const frame = try testing.createFrame();
+ defer testing.test_session.closeAllPages();
+
+ const env = frame.js.env;
+ defer env.cancelTerminate();
+
+ var watchdog = Watchdog.init(30_000);
+ var heartbeat: Heartbeat = .{};
+ var entry: Entry = .{ .env = env, .heartbeat = &heartbeat };
+ watchdog.register(&entry);
+ defer watchdog.unregister(&entry);
+
+ // The report's request time comes from the real clock, so the stall is
+ // placed in the past and scans run at (or just after) the real now.
+ const now = lp.datetime.milliTimestamp(.boot);
+ heartbeat.last_activity.store(now - 31_000, .release);
+
+ watchdog.scan(now - 2_000);
+ try testing.expectEqual(false, entry.fired);
+
+ watchdog.scan(now);
+ try testing.expectEqual(true, entry.fired);
+ try testing.expect(env.pendingStallReport() != null);
+
+ // Still pending, but not for long enough to call it native.
+ watchdog.scan(now + 2_000);
+ try testing.expectEqual(false, entry.native_reported);
+
+ watchdog.scan(now + 6_000);
+ try testing.expectEqual(true, entry.native_reported);
+
+ // Reported once per stall.
+ watchdog.scan(now + 7_000);
+
+ // Activity ends the stall.
+ heartbeat.last_activity.store(now + 7_000, .release);
+ watchdog.scan(now + 8_000);
+ try testing.expectEqual(false, entry.fired);
+}
+
+test "Watchdog: no native report once the script report has been logged" {
+ const frame = try testing.createFrame();
+ defer testing.test_session.closeAllPages();
+
+ const env = frame.js.env;
+ defer env.cancelTerminate();
+
+ var watchdog = Watchdog.init(30_000);
+ var heartbeat: Heartbeat = .{};
+ var entry: Entry = .{ .env = env, .heartbeat = &heartbeat };
+ watchdog.register(&entry);
+ defer watchdog.unregister(&entry);
+
+ const now = lp.datetime.milliTimestamp(.boot);
+ heartbeat.last_activity.store(now - 31_000, .release);
+ watchdog.scan(now);
+
+ // Stands in for the interrupt landing: the worker consumed the report.
+ _ = env.stall_report_requested_at.swap(0, .acq_rel);
+
+ watchdog.scan(now + 9_000);
+ try testing.expectEqual(false, entry.native_reported);
+}
diff --git a/src/browser/js/Env.zig b/src/browser/js/Env.zig
index 5cff3cf6c..a929e476d 100644
--- a/src/browser/js/Env.zig
+++ b/src/browser/js/Env.zig
@@ -115,6 +115,10 @@ terminate_requested: std.atomic.Value(bool) = .init(false),
// Set by the watchdog thread, consumed on the worker by terminateInterrupt so
// the stalled script can be identified before it is killed.
stall_report_requested_at: std.atomic.Value(u64) = .init(0),
+// How long the worker had stalled when the watchdog requested the report.
+// Published before stall_report_requested_at, so it's visible to whoever
+// observes that as non-zero.
+stall_report_stalled_ms: std.atomic.Value(u64) = .init(0),
// Set while a V8 context (or the isolate) is being disposed.
tearing_down: bool = false,
@@ -703,11 +707,19 @@ pub fn requestTerminate(self: *Env) void {
// Called from the watchdog thread. Like requestTerminate, but the worker also
// logs the running script's frame URL and stack before termination.
-pub fn requestTerminateForStall(self: *Env) void {
+pub fn requestTerminateForStall(self: *Env, stalled_ms: u64) void {
+ self.stall_report_stalled_ms.store(stalled_ms, .monotonic);
self.stall_report_requested_at.store(@max(lp.datetime.milliTimestamp(.boot), 1), .release);
self.requestTerminate();
}
+// Boot-clock ms at which a stall report was requested but not yet logged by
+// the worker, or null. Read by the watchdog thread.
+pub fn pendingStallReport(self: *const Env) ?u64 {
+ const requested_at = self.stall_report_requested_at.load(.acquire);
+ return if (requested_at == 0) null else requested_at;
+}
+
// Runs on the worker thread
fn terminateInterrupt(_: ?*v8.Isolate, data: ?*anyopaque) callconv(.c) void {
const self: *Env = @ptrCast(@alignCast(data.?));
@@ -760,7 +772,8 @@ fn stallReport(self: *Env, stack_buf: []u8) StallReport {
fn logStall(self: *Env, requested_at: u64) void {
var stack_buf: [1536]u8 = undefined;
const report = self.stallReport(&stack_buf);
- log.warn(.app, "watchdog stall script", .{
+ log.warn(.watchdog, "watchdog stall script", .{
+ .stalled_ms = self.stall_report_stalled_ms.load(.monotonic),
.url = report.url,
.page_url = report.page_url,
// Large when the stall was in native code: the interrupt only lands
@@ -958,7 +971,7 @@ test "Env: stall report names the running frame and script stack" {
}
test "Env: watchdog termination logs the stalled script once" {
- testing.expectLog(&.{.app});
+ testing.expectLog(&.{.watchdog});
const frame = try testing.createFrame();
defer testing.test_session.closeAllPages();
@@ -975,8 +988,8 @@ test "Env: watchdog termination logs the stalled script once" {
env: *Env,
fn stall(self: *@This()) void {
// Two in-flight interrupts must still produce one report.
- self.env.requestTerminateForStall();
- self.env.requestTerminateForStall();
+ self.env.requestTerminateForStall(31_000);
+ self.env.requestTerminateForStall(32_000);
}
};
var state = State{ .env = env };
@@ -993,8 +1006,10 @@ test "Env: canceling a termination drops its pending stall report" {
defer testing.test_session.closeAllPages();
const env = frame.js.env;
- env.requestTerminateForStall();
+ env.requestTerminateForStall(31_000);
+ try testing.expect(env.pendingStallReport() != null);
env.cancelTerminate();
+ try testing.expectEqual(null, env.pendingStallReport());
try testing.expectEqual(0, env.stall_report_requested_at.load(.acquire));
var ls: js.Local.Scope = undefined;
diff --git a/src/log.zig b/src/log.zig
index f5fc35092..87f2d3984 100644
--- a/src/log.zig
+++ b/src/log.zig
@@ -43,6 +43,7 @@ pub const Scope = enum {
storage,
telemetry,
unknown_prop,
+ watchdog,
websocket,
cors,
};