Skip to content

Lightpanda: the headless browser designed for AI and automation

feature: add watchdog for JS hangs

Karl Seguin6f641b1250de
diff --git a/src/App.zig b/src/App.zig
index acb06a10b..4f339803f 100644
--- a/src/App.zig
+++ b/src/App.zig
@@ -26,6 +26,7 @@ const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
const Storage = @import("storage/Storage.zig");
const Network = @import("network/Network.zig");
+const Watchdog = @import("Watchdog.zig");
pub const ArenaPool = @import("ArenaPool.zig");
const log = lp.log;
@@ -39,6 +40,7 @@ storage: Storage,
platform: Platform,
snapshot: Snapshot,
telemetry: Telemetry,
+watchdog: Watchdog,
allocator: Allocator,
arena_pool: ArenaPool,
app_dir_path: ?[]const u8,
@@ -66,7 +68,11 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
.app_dir_path = undefined,
.telemetry = undefined,
.arena_pool = undefined,
+ .watchdog = .init(config.watchdogMs()),
};
+ try app.watchdog.start();
+ errdefer app.watchdog.deinit();
+
app.network = try Network.init(allocator, app, config);
errdefer app.network.deinit();
@@ -87,6 +93,9 @@ pub fn shutdown(self: *const App) bool {
pub fn deinit(self: *App) void {
const allocator = self.allocator;
+ // All browsers are gone by now, so the entry list is empty; this just
+ // stops the checker thread.
+ self.watchdog.deinit();
if (self.app_dir_path) |app_dir_path| {
allocator.free(app_dir_path);
self.app_dir_path = null;
diff --git a/src/Config.zig b/src/Config.zig
index cc2a0e5ad..7b49c751a 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -116,6 +116,7 @@ const CommonOptions = .{
.{ .name = "enable_external_stylesheets", .type = bool },
.{ .name = "v8_flags_unsafe", .type = ?[]const u8 },
.{ .name = "v8_max_heap_mb", .type = ?u32 },
+ .{ .name = "watchdog_ms", .type = ?u32 },
};
fn dumpValidator(_: Allocator, args: *std.process.ArgIterator) !?DumpFormat {
@@ -340,6 +341,13 @@ pub fn disableWorkers(self: *const Config) bool {
};
}
+pub fn watchdogMs(self: *const Config) ?u32 {
+ return switch (self.mode) {
+ inline .serve, .fetch, .mcp, .agent => |opts| opts.watchdog_ms,
+ else => unreachable,
+ };
+}
+
pub fn enableExternalStylesheets(self: *const Config) bool {
return switch (self.mode) {
inline .serve, .fetch, .mcp, .agent => |opts| opts.enable_external_stylesheets,
diff --git a/src/Watchdog.zig b/src/Watchdog.zig
new file mode 100644
index 000000000..c205adaa9
--- /dev/null
+++ b/src/Watchdog.zig
@@ -0,0 +1,178 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const lp = @import("lightpanda");
+
+const Env = @import("browser/js/Env.zig");
+
+const log = lp.log;
+const milliTimestamp = @import("datetime.zig").milliTimestamp;
+
+// How often the checker thread scans the entries.
+const CHECK_INTERVAL_NS = 1 * std.time.ns_per_s;
+
+const Watchdog = @This();
+
+// null == disabled: no thread, register/unregister no-op.
+timeout_ms: ?u32,
+shutdown: bool = false,
+thread: ?std.Thread = null,
+mutex: std.Thread.Mutex = .{},
+cond: std.Thread.Condition = .{},
+entries: std.DoublyLinkedList = .{},
+
+// Embedded in Browser; must outlive the register/unregister window.
+pub const Entry = struct {
+ env: *Env,
+ heartbeat: *Heartbeat,
+ fired: bool = false,
+ registered: bool = false,
+ node: std.DoublyLinkedList.Node = .{},
+};
+
+pub fn init(timeout_ms: ?u32) Watchdog {
+ return .{ .timeout_ms = timeout_ms };
+}
+
+pub fn deinit(self: *Watchdog) void {
+ const thread = self.thread orelse return;
+ {
+ self.mutex.lock();
+ defer self.mutex.unlock();
+ self.shutdown = true;
+ self.cond.signal();
+ }
+ thread.join();
+}
+
+// Call once the Watchdog is at its final address (init returns by value).
+pub fn start(self: *Watchdog) !void {
+ if (self.timeout_ms == null) {
+ return;
+ }
+ self.thread = try std.Thread.spawn(.{}, run, .{self});
+}
+
+pub fn register(self: *Watchdog, entry: *Entry) void {
+ if (self.timeout_ms == null) {
+ return;
+ }
+
+ {
+ self.mutex.lock();
+ defer self.mutex.unlock();
+ self.entries.append(&entry.node);
+ }
+ entry.registered = true;
+}
+
+pub fn unregister(self: *Watchdog, entry: *Entry) void {
+ if (entry.registered == false) {
+ return;
+ }
+
+ {
+ self.mutex.lock();
+ defer self.mutex.unlock();
+ self.entries.remove(&entry.node);
+ }
+ entry.registered = false;
+}
+
+fn run(self: *Watchdog) void {
+ const timeout_ms: u64 = self.timeout_ms.?;
+
+ self.mutex.lock();
+ defer self.mutex.unlock();
+
+ while (true) {
+ self.cond.timedWait(&self.mutex, CHECK_INTERVAL_NS) catch {};
+ if (self.shutdown) {
+ return;
+ }
+
+ const now = milliTimestamp(.monotonic);
+ var node = self.entries.first;
+ while (node) |n| : (node = n.next) {
+ const entry: *Entry = @fieldParentPtr("node", n);
+ const heartbeat = entry.heartbeat;
+
+ if (heartbeat.wait_depth.load(.acquire) > 0) {
+ // The entry is in a controlled (e.g. non-JS) wait
+ entry.fired = false;
+ continue;
+ }
+
+ const last = heartbeat.last_activity.load(.acquire);
+ if (last == 0) {
+ // disarmed: no page work can be running
+ continue;
+ }
+
+ const stalled_ms = now -| last;
+ if (stalled_ms < timeout_ms) {
+ entry.fired = false;
+ continue;
+ }
+
+ if (entry.fired == false) {
+ entry.fired = true;
+ log.err(.app, "watchdog stall", .{ .stalled_ms = stalled_ms });
+ entry.env.requestTerminate();
+ }
+ }
+ }
+}
+
+// Written by the watched worker thread, read by the Watchdog thread.
+pub const Heartbeat = struct {
+ // > 0 while the worker is parked in a wait. Counterintuitive, but waiting
+ // can be nested (background task (wait_depth += 1) which runs microtask
+ // which does a syncRequest (wait_depth += 1). As long as we're waiting it
+ // means we aren't executing JavaScript and thus can't be in an endless JS
+ // loop.
+ wait_depth: std.atomic.Value(u32) = .init(0),
+
+ // The last time we saw some non-JS activity. 0 means disarmed: the worker
+ // is somewhere no page work can be running — before its first Runner tick
+ // (e.g. still in the CDP handshake read), or idle-pumping a session with
+ // no pages (MCP/agent between commands, see Session.idleSlice) — so the
+ // checker skips it.
+ last_activity: std.atomic.Value(u64) = .init(0),
+
+ pub fn touch(self: *Heartbeat) void {
+ self.last_activity.store(milliTimestamp(.monotonic), .release);
+ }
+
+ pub fn disarm(self: *Heartbeat) void {
+ self.last_activity.store(0, .release);
+ }
+
+ // Entering a planned wait (e.g. network poll)
+ pub fn enterWait(self: *Heartbeat) void {
+ self.touch();
+ _ = self.wait_depth.fetchAdd(1, .release);
+ }
+
+ // Existing a planned wait
+ pub fn exitWait(self: *Heartbeat) void {
+ self.touch();
+ _ = self.wait_depth.fetchSub(1, .release);
+ }
+};
diff --git a/src/browser/Browser.zig b/src/browser/Browser.zig
index 59c4867d7..e79834aa0 100644
--- a/src/browser/Browser.zig
+++ b/src/browser/Browser.zig
@@ -24,6 +24,7 @@ const Notification = @import("../Notification.zig");
const js = @import("js/js.zig");
const Page = @import("Page.zig");
+const Watchdog = @import("../Watchdog.zig");
const Session = @import("Session.zig");
const Selector = @import("webapi/selector/Selector.zig");
const Viewport = @import("Viewport.zig");
@@ -66,6 +67,10 @@ viewport_override: ?Viewport = null,
// used by sessions to allocate pages.
page_pool: std.heap.MemoryPool(Page),
+// Registered with App.watchdog for the lifetime of the env. The watchdog
+// checker thread reads it until Browser.deinit unregisters.
+watchdog_entry: Watchdog.Entry,
+
// Pool for FinalizerCallback.Identity structs — the records V8 weak-callback
// parameters point at. Scoped to the Browser (i.e. the V8 Isolate's lifetime)
// rather than the Session: V8 can run a weak finalizer arbitrarily late, any
@@ -114,13 +119,23 @@ pub fn init(self: *Browser, app: *App, opts: InitOpts, cdp: ?*CDP) !void {
.page_pool = std.heap.MemoryPool(Page).init(allocator),
.fc_identity_pool = .init(allocator),
.selector_cache = .init(allocator),
+ .watchdog_entry = undefined,
};
self.env.protectHeapLimit();
try self.http_client.init(allocator, &app.network, cdp);
+
+ self.watchdog_entry = .{
+ .env = &self.env,
+ .heartbeat = &self.http_client.heartbeat,
+ };
+ app.watchdog.register(&self.watchdog_entry);
}
pub fn deinit(self: *Browser) void {
self.closeSession();
+ // After this returns, the watchdog thread holds no reference to our env
+ // or http_client — required before either is torn down.
+ self.app.watchdog.unregister(&self.watchdog_entry);
self.env.deinit();
// After env.deinit() the Isolate is gone, so no further weak finalizer can
// fire — only now is it safe to free the pool backing their parameters.
diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig
index 06e2b22aa..7d5aedc16 100644
--- a/src/browser/HttpClient.zig
+++ b/src/browser/HttpClient.zig
@@ -32,6 +32,7 @@ const Network = @import("../network/Network.zig");
const CDP = @import("../cdp/CDP.zig");
const Inbox = @import("../Inbox.zig");
+const Watchdog = @import("../Watchdog.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
@@ -81,6 +82,12 @@ dirty: std.DoublyLinkedList = .{},
// Whether we're currently inside a curl_multi_perform call.
performing: bool = false,
+// Watchdog instrumentation for this client's worker thread. Wraps the poll
+// in perform (and the background-task wait in Runner) so the watchdog can
+// tell "parked, waiting for work" from "stuck between waits". Registered
+// with App.watchdog by Browser.init.
+heartbeat: Watchdog.Heartbeat = .{},
+
// Use to generate the next request ID
next_request_id: u32 = 0,
@@ -967,6 +974,8 @@ fn perform(self: *Client, timeout_ms: c_int) anyerror!void {
if (running > 0 or self.cdp_link_active) {
// when cdp_link_active == true, the network thread will unblock this
// by calling wakup on our multi.
+ self.heartbeat.enterWait();
+ defer self.heartbeat.exitWait();
try self.handles.poll(&.{}, timeout_ms);
}
diff --git a/src/browser/Runner.zig b/src/browser/Runner.zig
index eac23a500..a96adedf4 100644
--- a/src/browser/Runner.zig
+++ b/src/browser/Runner.zig
@@ -205,6 +205,10 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Wa
const browser = self.browser;
const http_client = self.http_client;
+ // Arms the watchdog (and proves liveness): a stall anywhere in this tick
+ // ages this stamp until the watchdog fires.
+ http_client.heartbeat.touch();
+
// Drain queued navigations across every live page (one page per call)
// A navigation can swap a frame pointer or the page set,
// so restart the tick to re-resolve cleanly.
@@ -229,6 +233,10 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Wa
// wait for them. Don't do this for CDP, since new CDP messages can always
// come in at any time.
if ((comptime is_cdp) == false and network_idle and browser.hasBackgroundTasks()) {
+ // We _are_ calling into v8, but these background tasks are v8-specific,
+ // meaning v8 is working, not stuck. We don't want the watchdog to kill this.
+ http_client.heartbeat.enterWait();
+ defer http_client.heartbeat.exitWait();
browser.waitForBackgroundTasks();
return .{ .ok = 0 };
}
diff --git a/src/browser/Session.zig b/src/browser/Session.zig
index 4d6d67918..d348ce9d7 100644
--- a/src/browser/Session.zig
+++ b/src/browser/Session.zig
@@ -493,6 +493,8 @@ pub fn idleSlice(self: *Session) u31 {
self.processDestroyQueues();
if (self.pages.items.len == 0) {
+ // no page, we don't want the watchdog to kill us.
+ self.browser.http_client.heartbeat.disarm();
return quiet_ms;
}
const page = self.pages.items[0];
diff --git a/src/browser/js/Env.zig b/src/browser/js/Env.zig
index 6edda76bd..b6c0fe207 100644
--- a/src/browser/js/Env.zig
+++ b/src/browser/js/Env.zig
@@ -399,6 +399,9 @@ pub fn runMicrotasks(self: *Env) void {
const v8_isolate = self.isolate.handle;
+ // terminatePending: once a forcible terminate is requested (and not
+ // canceled), refuse to start new work — IsExecutionTerminating alone
+ // goes false again as soon as the killed script finishes unwinding.
if (v8.v8__Isolate__IsExecutionTerminating(v8_isolate) or self.terminatePending()) {
return;
}
diff --git a/src/help.zon b/src/help.zon
index 08eb8c7fc..625a9cf86 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -354,6 +354,13 @@
\\ Maximum V8 heap size in megabytes, per browser instance. Values
\\ below ~16 are clamped by V8 and all behave the same.
\\ Defaults to the V8 default (based on available memory).
+ \\ --watchdog-ms <INT>
+ \\ Terminates JavaScript when the browser stalls — stays busy for
+ \\ this long without returning to its network poll (e.g. a page
+ \\ stuck in an endless script loop). In serve mode the CDP
+ \\ connection is then closed. Unlike fetch's --terminate-ms, an idle
+ \\ or normally-working page never trips this.
+ \\ Defaults to disabled.
\\ --web-bot-auth-domain <DOMAIN>
\\ Your domain, e.g. yourdomain.com.
\\ --web-bot-auth-key-file <PATH>