Lightpanda: the headless browser designed for AI and automation
feature: add watchdog for JS hangs
Karl Seguin6f641b1250de
diff --git a/src/App.zig b/src/App.zigindex acb06a10b..4f339803f 100644--- a/src/App.zig+++ b/src/App.zig@@ -26,6 +26,7 @@ const Telemetry = @import("telemetry/telemetry.zig").Telemetry;const Storage = @import("storage/Storage.zig");const Network = @import("network/Network.zig");+const Watchdog = @import("Watchdog.zig");pub const ArenaPool = @import("ArenaPool.zig");const log = lp.log;@@ -39,6 +40,7 @@ storage: Storage,platform: Platform,snapshot: Snapshot,telemetry: Telemetry,+watchdog: Watchdog,allocator: Allocator,arena_pool: ArenaPool,app_dir_path: ?[]const u8,@@ -66,7 +68,11 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {.app_dir_path = undefined,.telemetry = undefined,.arena_pool = undefined,+ .watchdog = .init(config.watchdogMs()),};+ try app.watchdog.start();+ errdefer app.watchdog.deinit();+app.network = try Network.init(allocator, app, config);errdefer app.network.deinit();@@ -87,6 +93,9 @@ pub fn shutdown(self: *const App) bool {pub fn deinit(self: *App) void {const allocator = self.allocator;+ // All browsers are gone by now, so the entry list is empty; this just+ // stops the checker thread.+ self.watchdog.deinit();if (self.app_dir_path) |app_dir_path| {allocator.free(app_dir_path);self.app_dir_path = null;diff --git a/src/Config.zig b/src/Config.zigindex cc2a0e5ad..7b49c751a 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -116,6 +116,7 @@ const CommonOptions = .{.{ .name = "enable_external_stylesheets", .type = bool },.{ .name = "v8_flags_unsafe", .type = ?[]const u8 },.{ .name = "v8_max_heap_mb", .type = ?u32 },+ .{ .name = "watchdog_ms", .type = ?u32 },};fn dumpValidator(_: Allocator, args: *std.process.ArgIterator) !?DumpFormat {@@ -340,6 +341,13 @@ pub fn disableWorkers(self: *const Config) bool {};}+pub fn watchdogMs(self: *const Config) ?u32 {+ return switch (self.mode) {+ inline .serve, .fetch, .mcp, .agent => |opts| opts.watchdog_ms,+ else => unreachable,+ };+}+pub fn enableExternalStylesheets(self: *const Config) bool {return switch (self.mode) {inline .serve, .fetch, .mcp, .agent => |opts| opts.enable_external_stylesheets,diff --git a/src/Watchdog.zig b/src/Watchdog.zignew file mode 100644index 000000000..c205adaa9--- /dev/null+++ b/src/Watchdog.zig@@ -0,0 +1,178 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const std = @import("std");+const lp = @import("lightpanda");++const Env = @import("browser/js/Env.zig");++const log = lp.log;+const milliTimestamp = @import("datetime.zig").milliTimestamp;++// How often the checker thread scans the entries.+const CHECK_INTERVAL_NS = 1 * std.time.ns_per_s;++const Watchdog = @This();++// null == disabled: no thread, register/unregister no-op.+timeout_ms: ?u32,+shutdown: bool = false,+thread: ?std.Thread = null,+mutex: std.Thread.Mutex = .{},+cond: std.Thread.Condition = .{},+entries: std.DoublyLinkedList = .{},++// Embedded in Browser; must outlive the register/unregister window.+pub const Entry = struct {+ env: *Env,+ heartbeat: *Heartbeat,+ fired: bool = false,+ registered: bool = false,+ node: std.DoublyLinkedList.Node = .{},+};++pub fn init(timeout_ms: ?u32) Watchdog {+ return .{ .timeout_ms = timeout_ms };+}++pub fn deinit(self: *Watchdog) void {+ const thread = self.thread orelse return;+ {+ self.mutex.lock();+ defer self.mutex.unlock();+ self.shutdown = true;+ self.cond.signal();+ }+ thread.join();+}++// Call once the Watchdog is at its final address (init returns by value).+pub fn start(self: *Watchdog) !void {+ if (self.timeout_ms == null) {+ return;+ }+ self.thread = try std.Thread.spawn(.{}, run, .{self});+}++pub fn register(self: *Watchdog, entry: *Entry) void {+ if (self.timeout_ms == null) {+ return;+ }++ {+ self.mutex.lock();+ defer self.mutex.unlock();+ self.entries.append(&entry.node);+ }+ entry.registered = true;+}++pub fn unregister(self: *Watchdog, entry: *Entry) void {+ if (entry.registered == false) {+ return;+ }++ {+ self.mutex.lock();+ defer self.mutex.unlock();+ self.entries.remove(&entry.node);+ }+ entry.registered = false;+}++fn run(self: *Watchdog) void {+ const timeout_ms: u64 = self.timeout_ms.?;++ self.mutex.lock();+ defer self.mutex.unlock();++ while (true) {+ self.cond.timedWait(&self.mutex, CHECK_INTERVAL_NS) catch {};+ if (self.shutdown) {+ return;+ }++ const now = milliTimestamp(.monotonic);+ var node = self.entries.first;+ while (node) |n| : (node = n.next) {+ const entry: *Entry = @fieldParentPtr("node", n);+ const heartbeat = entry.heartbeat;++ if (heartbeat.wait_depth.load(.acquire) > 0) {+ // The entry is in a controlled (e.g. non-JS) wait+ entry.fired = false;+ continue;+ }++ const last = heartbeat.last_activity.load(.acquire);+ if (last == 0) {+ // disarmed: no page work can be running+ continue;+ }++ const stalled_ms = now -| last;+ if (stalled_ms < timeout_ms) {+ entry.fired = false;+ continue;+ }++ if (entry.fired == false) {+ entry.fired = true;+ log.err(.app, "watchdog stall", .{ .stalled_ms = stalled_ms });+ entry.env.requestTerminate();+ }+ }+ }+}++// Written by the watched worker thread, read by the Watchdog thread.+pub const Heartbeat = struct {+ // > 0 while the worker is parked in a wait. Counterintuitive, but waiting+ // can be nested (background task (wait_depth += 1) which runs microtask+ // which does a syncRequest (wait_depth += 1). As long as we're waiting it+ // means we aren't executing JavaScript and thus can't be in an endless JS+ // loop.+ wait_depth: std.atomic.Value(u32) = .init(0),++ // The last time we saw some non-JS activity. 0 means disarmed: the worker+ // is somewhere no page work can be running — before its first Runner tick+ // (e.g. still in the CDP handshake read), or idle-pumping a session with+ // no pages (MCP/agent between commands, see Session.idleSlice) — so the+ // checker skips it.+ last_activity: std.atomic.Value(u64) = .init(0),++ pub fn touch(self: *Heartbeat) void {+ self.last_activity.store(milliTimestamp(.monotonic), .release);+ }++ pub fn disarm(self: *Heartbeat) void {+ self.last_activity.store(0, .release);+ }++ // Entering a planned wait (e.g. network poll)+ pub fn enterWait(self: *Heartbeat) void {+ self.touch();+ _ = self.wait_depth.fetchAdd(1, .release);+ }++ // Existing a planned wait+ pub fn exitWait(self: *Heartbeat) void {+ self.touch();+ _ = self.wait_depth.fetchSub(1, .release);+ }+};diff --git a/src/browser/Browser.zig b/src/browser/Browser.zigindex 59c4867d7..e79834aa0 100644--- a/src/browser/Browser.zig+++ b/src/browser/Browser.zig@@ -24,6 +24,7 @@ const Notification = @import("../Notification.zig");const js = @import("js/js.zig");const Page = @import("Page.zig");+const Watchdog = @import("../Watchdog.zig");const Session = @import("Session.zig");const Selector = @import("webapi/selector/Selector.zig");const Viewport = @import("Viewport.zig");@@ -66,6 +67,10 @@ viewport_override: ?Viewport = null,// used by sessions to allocate pages.page_pool: std.heap.MemoryPool(Page),+// Registered with App.watchdog for the lifetime of the env. The watchdog+// checker thread reads it until Browser.deinit unregisters.+watchdog_entry: Watchdog.Entry,+// Pool for FinalizerCallback.Identity structs — the records V8 weak-callback// parameters point at. Scoped to the Browser (i.e. the V8 Isolate's lifetime)// rather than the Session: V8 can run a weak finalizer arbitrarily late, any@@ -114,13 +119,23 @@ pub fn init(self: *Browser, app: *App, opts: InitOpts, cdp: ?*CDP) !void {.page_pool = std.heap.MemoryPool(Page).init(allocator),.fc_identity_pool = .init(allocator),.selector_cache = .init(allocator),+ .watchdog_entry = undefined,};self.env.protectHeapLimit();try self.http_client.init(allocator, &app.network, cdp);++ self.watchdog_entry = .{+ .env = &self.env,+ .heartbeat = &self.http_client.heartbeat,+ };+ app.watchdog.register(&self.watchdog_entry);}pub fn deinit(self: *Browser) void {self.closeSession();+ // After this returns, the watchdog thread holds no reference to our env+ // or http_client — required before either is torn down.+ self.app.watchdog.unregister(&self.watchdog_entry);self.env.deinit();// After env.deinit() the Isolate is gone, so no further weak finalizer can// fire — only now is it safe to free the pool backing their parameters.diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zigindex 06e2b22aa..7d5aedc16 100644--- a/src/browser/HttpClient.zig+++ b/src/browser/HttpClient.zig@@ -32,6 +32,7 @@ const Network = @import("../network/Network.zig");const CDP = @import("../cdp/CDP.zig");const Inbox = @import("../Inbox.zig");+const Watchdog = @import("../Watchdog.zig");const log = lp.log;const Allocator = std.mem.Allocator;@@ -81,6 +82,12 @@ dirty: std.DoublyLinkedList = .{},// Whether we're currently inside a curl_multi_perform call.performing: bool = false,+// Watchdog instrumentation for this client's worker thread. Wraps the poll+// in perform (and the background-task wait in Runner) so the watchdog can+// tell "parked, waiting for work" from "stuck between waits". Registered+// with App.watchdog by Browser.init.+heartbeat: Watchdog.Heartbeat = .{},+// Use to generate the next request IDnext_request_id: u32 = 0,@@ -967,6 +974,8 @@ fn perform(self: *Client, timeout_ms: c_int) anyerror!void {if (running > 0 or self.cdp_link_active) {// when cdp_link_active == true, the network thread will unblock this// by calling wakup on our multi.+ self.heartbeat.enterWait();+ defer self.heartbeat.exitWait();try self.handles.poll(&.{}, timeout_ms);}diff --git a/src/browser/Runner.zig b/src/browser/Runner.zigindex eac23a500..a96adedf4 100644--- a/src/browser/Runner.zig+++ b/src/browser/Runner.zig@@ -205,6 +205,10 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Waconst browser = self.browser;const http_client = self.http_client;+ // Arms the watchdog (and proves liveness): a stall anywhere in this tick+ // ages this stamp until the watchdog fires.+ http_client.heartbeat.touch();+// Drain queued navigations across every live page (one page per call)// A navigation can swap a frame pointer or the page set,// so restart the tick to re-resolve cleanly.@@ -229,6 +233,10 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Wa// wait for them. Don't do this for CDP, since new CDP messages can always// come in at any time.if ((comptime is_cdp) == false and network_idle and browser.hasBackgroundTasks()) {+ // We _are_ calling into v8, but these background tasks are v8-specific,+ // meaning v8 is working, not stuck. We don't want the watchdog to kill this.+ http_client.heartbeat.enterWait();+ defer http_client.heartbeat.exitWait();browser.waitForBackgroundTasks();return .{ .ok = 0 };}diff --git a/src/browser/Session.zig b/src/browser/Session.zigindex 4d6d67918..d348ce9d7 100644--- a/src/browser/Session.zig+++ b/src/browser/Session.zig@@ -493,6 +493,8 @@ pub fn idleSlice(self: *Session) u31 {self.processDestroyQueues();if (self.pages.items.len == 0) {+ // no page, we don't want the watchdog to kill us.+ self.browser.http_client.heartbeat.disarm();return quiet_ms;}const page = self.pages.items[0];diff --git a/src/browser/js/Env.zig b/src/browser/js/Env.zigindex 6edda76bd..b6c0fe207 100644--- a/src/browser/js/Env.zig+++ b/src/browser/js/Env.zig@@ -399,6 +399,9 @@ pub fn runMicrotasks(self: *Env) void {const v8_isolate = self.isolate.handle;+ // terminatePending: once a forcible terminate is requested (and not+ // canceled), refuse to start new work — IsExecutionTerminating alone+ // goes false again as soon as the killed script finishes unwinding.if (v8.v8__Isolate__IsExecutionTerminating(v8_isolate) or self.terminatePending()) {return;}diff --git a/src/help.zon b/src/help.zonindex 08eb8c7fc..625a9cf86 100644--- a/src/help.zon+++ b/src/help.zon@@ -354,6 +354,13 @@\\ Maximum V8 heap size in megabytes, per browser instance. Values\\ below ~16 are clamped by V8 and all behave the same.\\ Defaults to the V8 default (based on available memory).+ \\ --watchdog-ms <INT>+ \\ Terminates JavaScript when the browser stalls — stays busy for+ \\ this long without returning to its network poll (e.g. a page+ \\ stuck in an endless script loop). In serve mode the CDP+ \\ connection is then closed. Unlike fetch's --terminate-ms, an idle+ \\ or normally-working page never trips this.+ \\ Defaults to disabled.\\ --web-bot-auth-domain <DOMAIN>\\ Your domain, e.g. yourdomain.com.\\ --web-bot-auth-key-file <PATH>