Lightpanda: the headless browser designed for AI and automation
Warn from the watchdog thread when a stall never returns to JavaScript
Scott Taylorbfcd5ebd280f
diff --git a/src/Watchdog.zig b/src/Watchdog.zigindex abbf3b07b..f3bfcaafd 100644--- a/src/Watchdog.zig+++ b/src/Watchdog.zig@@ -26,6 +26,12 @@ const log = lp.log;// How often the checker thread scans the entries.const CHECK_INTERVAL_NS = 1 * std.time.ns_per_s;+// The "watchdog stall script" line is logged by the termination interrupt,+// which only runs once the worker is back in JavaScript. If it's still pending+// this long after the stall was detected, the worker is stuck in native code:+// say so from here, or the stall leaves no trace above debug.+const NATIVE_STALL_REPORT_MS = 5_000;+const Watchdog = @This();// null == disabled: no thread, register/unregister no-op.@@ -41,6 +47,9 @@ pub const Entry = struct {env: *Env,heartbeat: *Heartbeat,fired: bool = false,+ // Set once the current stall has been reported as native (see+ // NATIVE_STALL_REPORT_MS). Reset when a new stall fires.+ native_reported: bool = false,registered: bool = false,node: std.DoublyLinkedList.Node = .{},};@@ -95,8 +104,6 @@ pub fn unregister(self: *Watchdog, entry: *Entry) void {}fn run(self: *Watchdog) void {- const timeout_ms: u64 = self.timeout_ms.?;-self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);@@ -105,38 +112,58 @@ fn run(self: *Watchdog) void {if (self.shutdown) {return;}+ self.scan(lp.datetime.milliTimestamp(.boot));+ }+}++// Called with the mutex held.+fn scan(self: *Watchdog, now: u64) void {+ const timeout_ms: u64 = self.timeout_ms.?;++ var node = self.entries.first;+ while (node) |n| : (node = n.next) {+ const entry: *Entry = @fieldParentPtr("node", n);+ const heartbeat = entry.heartbeat;++ if (heartbeat.wait_depth.load(.acquire) > 0) {+ // The entry is in a controlled (e.g. non-JS) wait+ entry.fired = false;+ continue;+ }++ const last = heartbeat.last_activity.load(.acquire);+ if (last == 0) {+ // disarmed: no page work can be running+ continue;+ }++ const stalled_ms = now -| last;+ if (stalled_ms < timeout_ms) {+ entry.fired = false;+ continue;+ }- const now = lp.datetime.milliTimestamp(.boot);- var node = self.entries.first;- while (node) |n| : (node = n.next) {- const entry: *Entry = @fieldParentPtr("node", n);- const heartbeat = entry.heartbeat;-- if (heartbeat.wait_depth.load(.acquire) > 0) {- // The entry is in a controlled (e.g. non-JS) wait- entry.fired = false;- continue;- }-- const last = heartbeat.last_activity.load(.acquire);- if (last == 0) {- // disarmed: no page work can be running- continue;- }-- const stalled_ms = now -| last;- if (stalled_ms < timeout_ms) {- entry.fired = false;- continue;- }-- if (entry.fired == false) {- entry.fired = true;- log.debug(.app, "watchdog stall", .{ .stalled_ms = stalled_ms });- // Also logs "watchdog stall script" (URL and JS stack) from- // the worker thread when the termination interrupt lands.- entry.env.requestTerminateForStall();- }+ if (entry.fired == false) {+ entry.fired = true;+ entry.native_reported = false;+ log.debug(.watchdog, "watchdog stall", .{ .stalled_ms = stalled_ms });+ // The worker logs "watchdog stall script" (URL and JS stack) when+ // the termination interrupt lands.+ entry.env.requestTerminateForStall(stalled_ms);+ continue;+ }++ if (entry.native_reported) {+ continue;+ }+ const requested_at = entry.env.pendingStallReport() orelse continue;+ const pending_ms = now -| requested_at;+ if (pending_ms >= NATIVE_STALL_REPORT_MS) {+ entry.native_reported = true;+ log.warn(.watchdog, "watchdog stall native", .{+ .stalled_ms = stalled_ms,+ .pending_ms = pending_ms,+ });}}}@@ -177,3 +204,73 @@ pub const Heartbeat = struct {_ = self.wait_depth.fetchSub(1, .release);}};++const testing = @import("testing.zig");+test "Watchdog: a stall that never returns to JavaScript is reported once" {+ // Only the native warn: the first line is debug, and the termination+ // interrupt never lands because no JavaScript runs.+ testing.expectLog(&.{.watchdog});++ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();++ const env = frame.js.env;+ defer env.cancelTerminate();++ var watchdog = Watchdog.init(30_000);+ var heartbeat: Heartbeat = .{};+ var entry: Entry = .{ .env = env, .heartbeat = &heartbeat };+ watchdog.register(&entry);+ defer watchdog.unregister(&entry);++ // The report's request time comes from the real clock, so the stall is+ // placed in the past and scans run at (or just after) the real now.+ const now = lp.datetime.milliTimestamp(.boot);+ heartbeat.last_activity.store(now - 31_000, .release);++ watchdog.scan(now - 2_000);+ try testing.expectEqual(false, entry.fired);++ watchdog.scan(now);+ try testing.expectEqual(true, entry.fired);+ try testing.expect(env.pendingStallReport() != null);++ // Still pending, but not for long enough to call it native.+ watchdog.scan(now + 2_000);+ try testing.expectEqual(false, entry.native_reported);++ watchdog.scan(now + 6_000);+ try testing.expectEqual(true, entry.native_reported);++ // Reported once per stall.+ watchdog.scan(now + 7_000);++ // Activity ends the stall.+ heartbeat.last_activity.store(now + 7_000, .release);+ watchdog.scan(now + 8_000);+ try testing.expectEqual(false, entry.fired);+}++test "Watchdog: no native report once the script report has been logged" {+ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();++ const env = frame.js.env;+ defer env.cancelTerminate();++ var watchdog = Watchdog.init(30_000);+ var heartbeat: Heartbeat = .{};+ var entry: Entry = .{ .env = env, .heartbeat = &heartbeat };+ watchdog.register(&entry);+ defer watchdog.unregister(&entry);++ const now = lp.datetime.milliTimestamp(.boot);+ heartbeat.last_activity.store(now - 31_000, .release);+ watchdog.scan(now);++ // Stands in for the interrupt landing: the worker consumed the report.+ _ = env.stall_report_requested_at.swap(0, .acq_rel);++ watchdog.scan(now + 9_000);+ try testing.expectEqual(false, entry.native_reported);+}diff --git a/src/browser/js/Env.zig b/src/browser/js/Env.zigindex 5cff3cf6c..a929e476d 100644--- a/src/browser/js/Env.zig+++ b/src/browser/js/Env.zig@@ -115,6 +115,10 @@ terminate_requested: std.atomic.Value(bool) = .init(false),// Set by the watchdog thread, consumed on the worker by terminateInterrupt so// the stalled script can be identified before it is killed.stall_report_requested_at: std.atomic.Value(u64) = .init(0),+// How long the worker had stalled when the watchdog requested the report.+// Published before stall_report_requested_at, so it's visible to whoever+// observes that as non-zero.+stall_report_stalled_ms: std.atomic.Value(u64) = .init(0),// Set while a V8 context (or the isolate) is being disposed.tearing_down: bool = false,@@ -703,11 +707,19 @@ pub fn requestTerminate(self: *Env) void {// Called from the watchdog thread. Like requestTerminate, but the worker also// logs the running script's frame URL and stack before termination.-pub fn requestTerminateForStall(self: *Env) void {+pub fn requestTerminateForStall(self: *Env, stalled_ms: u64) void {+ self.stall_report_stalled_ms.store(stalled_ms, .monotonic);self.stall_report_requested_at.store(@max(lp.datetime.milliTimestamp(.boot), 1), .release);self.requestTerminate();}+// Boot-clock ms at which a stall report was requested but not yet logged by+// the worker, or null. Read by the watchdog thread.+pub fn pendingStallReport(self: *const Env) ?u64 {+ const requested_at = self.stall_report_requested_at.load(.acquire);+ return if (requested_at == 0) null else requested_at;+}+// Runs on the worker threadfn terminateInterrupt(_: ?*v8.Isolate, data: ?*anyopaque) callconv(.c) void {const self: *Env = @ptrCast(@alignCast(data.?));@@ -760,7 +772,8 @@ fn stallReport(self: *Env, stack_buf: []u8) StallReport {fn logStall(self: *Env, requested_at: u64) void {var stack_buf: [1536]u8 = undefined;const report = self.stallReport(&stack_buf);- log.warn(.app, "watchdog stall script", .{+ log.warn(.watchdog, "watchdog stall script", .{+ .stalled_ms = self.stall_report_stalled_ms.load(.monotonic),.url = report.url,.page_url = report.page_url,// Large when the stall was in native code: the interrupt only lands@@ -958,7 +971,7 @@ test "Env: stall report names the running frame and script stack" {}test "Env: watchdog termination logs the stalled script once" {- testing.expectLog(&.{.app});+ testing.expectLog(&.{.watchdog});const frame = try testing.createFrame();defer testing.test_session.closeAllPages();@@ -975,8 +988,8 @@ test "Env: watchdog termination logs the stalled script once" {env: *Env,fn stall(self: *@This()) void {// Two in-flight interrupts must still produce one report.- self.env.requestTerminateForStall();- self.env.requestTerminateForStall();+ self.env.requestTerminateForStall(31_000);+ self.env.requestTerminateForStall(32_000);}};var state = State{ .env = env };@@ -993,8 +1006,10 @@ test "Env: canceling a termination drops its pending stall report" {defer testing.test_session.closeAllPages();const env = frame.js.env;- env.requestTerminateForStall();+ env.requestTerminateForStall(31_000);+ try testing.expect(env.pendingStallReport() != null);env.cancelTerminate();+ try testing.expectEqual(null, env.pendingStallReport());try testing.expectEqual(0, env.stall_report_requested_at.load(.acquire));var ls: js.Local.Scope = undefined;diff --git a/src/log.zig b/src/log.zigindex f5fc35092..87f2d3984 100644--- a/src/log.zig+++ b/src/log.zig@@ -43,6 +43,7 @@ pub const Scope = enum {storage,telemetry,unknown_prop,+ watchdog,websocket,cors,};