Skip to content

Lightpanda: the headless browser designed for AI and automation

have httpclient own idle time

Karl Seguin111fd35258f9
diff --git a/src/browser/Runner.zig b/src/browser/Runner.zig
index 143f04adc..8cb55745b 100644
--- a/src/browser/Runner.zig
+++ b/src/browser/Runner.zig
@@ -37,11 +37,6 @@ browser: *Browser,
http_client: *HttpClient,
background_poll_ms: u32 = 0,
-// Not Frame._notified_network_idle: that latches for CDP's one-shot lifecycle
-// event, so a frame idle once reads idle forever.
-network_idle_since: ?u64 = null,
-network_almost_idle_since: ?u64 = null,
-
const network_idle_hold_ms = 500;
pub const Opts = struct {};
@@ -226,8 +221,6 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Wa
// A navigation can swap a frame pointer or the page set,
// so restart the tick to re-resolve cleanly.
if (try session.processQueuedNavigation()) {
- self.network_idle_since = null;
- self.network_almost_idle_since = null;
return .{ .ok = 0 };
}
@@ -244,9 +237,12 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Wa
const network_idle = activity.idle();
const is_done = browser.hasMacrotasks() == false and network_idle;
- const now = lp.datetime.milliTimestamp(.boot);
- const network_idle_held = holdsFor(&self.network_idle_since, network_idle, now);
- const network_almost_idle_held = holdsFor(&self.network_almost_idle_since, activity.total() <= 2, now);
+ // Not Frame._notified_network_idle: that latches for CDP's one-shot
+ // lifecycle event, so a frame idle once reads idle forever.
+ const network_idle_held, const network_almost_idle_held = blk: {
+ const http_idle, const http_almost_idle = http_client.idleMs();
+ break :blk .{ http_idle >= network_idle_hold_ms, http_almost_idle >= network_idle_hold_ms };
+ };
// Outside the condition loop: it skips resolved conditions, but an idle
// notification needs a check 500ms+ after the hold starts, and on a quiet
@@ -367,18 +363,6 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Wa
return .done;
}
-fn holdsFor(since: *?u64, active: bool, now: u64) bool {
- if (active == false) {
- since.* = null;
- return false;
- }
- const start = since.* orelse {
- since.* = now;
- return false;
- };
- return now - start >= network_idle_hold_ms;
-}
-
pub fn waitForSelector(self: *Runner, frame_id: u32, input: [:0]const u8, timeout_ms: u32) !*Node.Element {
const session = self.session;
const arena = try session.getArena(.small, "Runner.waitForSelector");
@@ -693,4 +677,10 @@ test "Runner: networkidle waits out activity after the frame latched idle" {
try runner.waitForFrame(page.frame_id, 3000, .{ .until = .networkidle });
try testing.expectEqual(true, lp.datetime.milliTimestamp(.boot) - start >= 500);
_ = try runner.waitForSelector(page.frame_id, "#fetched", 0);
+
+ // Already idle past the hold: a fresh wait doesn't serve it again.
+ var again = page.session.runner(.{});
+ const again_start = lp.datetime.milliTimestamp(.boot);
+ try again.waitForFrame(page.frame_id, 3000, .{ .until = .networkidle });
+ try testing.expectEqual(true, lp.datetime.milliTimestamp(.boot) - again_start < 250);
}
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 5f62fb0bf..decf406a4 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -68,6 +68,12 @@ ws_active: usize = 0,
// Count of active http requests
http_active: usize = 0,
+// Since when http activity has been 0 / at most 2, backing networkidle and
+// networkalmostidle. Sampled each tick and cleared when a transfer is created,
+// so one that starts and finishes between two samples still breaks the hold.
+idle_since: ?u64 = null,
+almost_idle_since: ?u64 = null,
+
// Our curl multi handle.
handles: http.Handles,
@@ -608,6 +614,35 @@ pub fn activity(self: *const Client) Activity {
};
}
+// How long http activity has held at 0 and at most 2, each 0 if it doesn't
+// now. WebSockets don't count: an open socket would never let a page go idle.
+pub fn idleMs(self: *Client) struct { u64, u64 } {
+ const now = lp.datetime.milliTimestamp(.boot);
+ self.sampleIdle(now);
+
+ return .{
+ now - (self.idle_since orelse now),
+ now - (self.almost_idle_since orelse now),
+ };
+}
+
+fn sampleIdle(self: *Client, now: u64) void {
+ const a = self.activity();
+ if (a.pending or a.http > 2) {
+ self.idle_since = null;
+ self.almost_idle_since = null;
+ return;
+ }
+ if (self.almost_idle_since == null) {
+ self.almost_idle_since = now;
+ }
+ if (a.http > 0) {
+ self.idle_since = null;
+ } else if (self.idle_since == null) {
+ self.idle_since = now;
+ }
+}
+
// What client messages drainInbox is allowed to dispatch this tick.
// .all — outer event loop (Runner.tick). Safe to dispatch
// everything; the JS stack is empty.
@@ -717,6 +752,11 @@ pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfe
return err;
};
+ self.idle_since = null;
+ if (self.activity().http >= 2) {
+ self.almost_idle_since = null;
+ }
+
if (owner) |o| {
o.addTransfer(transfer);
transfer.owner = o;
@@ -756,6 +796,8 @@ pub fn _tick(self: *Client, timeout_ms: u32, mode: DrainMode) !bool {
return error.ClientDisconnected;
}
+ defer self.sampleIdle(lp.datetime.milliTimestamp(.boot));
+
var waited = true;
const dispatched = self.dispatchCompleted(mode);