Lightpanda: the headless browser designed for AI and automation
ops: extend PageStatus to include fetch statistics
Karl Seguinf499fbc84f66
diff --git a/src/browser/PageStatus.zig b/src/browser/PageStatus.zigindex bc6b193cf..990ed25d6 100644--- a/src/browser/PageStatus.zig+++ b/src/browser/PageStatus.zig@@ -19,6 +19,9 @@const std = @import("std");const lp = @import("lightpanda");+const http = @import("../network/http.zig");+const HttpClient = @import("../network/HttpClient.zig");+const log = lp.log;const PageStatus = @This();@@ -35,9 +38,25 @@ pub const Skipped = enum {stylesheet,};+// The sub-resources the page itself asks for, and which we load.+pub const Loaded = enum {+ script,+ fetch,+ xhr,+};+js_errors: u32 = 0,skipped: Counts(Skipped) = .{},unsupported: Counts(Unsupported) = .{},+// Sub-resource loads that got a status >= 400 or a network error. Aborts and+// requests we refused ourselves (robots, --block-urls, adblock) aren't the+// site failing, so they don't count.+failed: Counts(Loaded) = .{},+// The 403s and 429s among `failed`: a bot wall or rate limit on the page's+// own API calls often leaves the document itself a 200.+blocked: Counts(Loaded) = .{},+// The first recognized bot challenge on a sub-resource.+resource_challenge: ?HttpClient.BotChallenge = null,// Globals that uncaught ReferenceErrors named. Most of what a page needs and// we lack shows up here first, before any call we could count: TradingView// dies at `new Path2D()` and never reaches getContext. Deduped and capped;@@ -107,6 +126,43 @@ pub fn noteSkipped(self: *PageStatus, what: Skipped) void {}}+// A sub-resource's final response headers.+pub fn noteResponse(self: *PageStatus, what: Loaded, transfer: *const HttpClient.Transfer) void {+ const status = transfer.responseStatus() orelse return;++ // Before the status check: AWS WAF challenges with a 202.+ if (self.resource_challenge == null) {+ if (transfer.botChallenge()) |challenge| {+ self.resource_challenge = challenge;+ log.warn(.http, "sub-resource bot challenge", .{ .challenge = challenge, .resource = what, .url = transfer.req.url });+ }+ }++ if (status < 400) {+ return;+ }+ _ = self.failed.incr(what);++ if (status == 403 or status == 429) {+ if (self.blocked.incr(what) == 1) {+ log.info(.http, "sub-resource blocked", .{ .resource = what, .status = status, .url = transfer.req.url });+ }+ }+}++// A sub-resource load that ended without a usable response. Callers only+// report errors that came before any response headers, which noteResponse+// has already counted.+pub fn noteError(self: *PageStatus, what: Loaded, err: anyerror) void {+ if (err == error.UrlBlocked) {+ return;+ }+ switch (http.errorReason(err)) {+ .aborted, .robots_blocked => return,+ else => _ = self.failed.incr(what),+ }+}+pub const MissingGlobals = struct {const max = 8;const max_len = 48;@@ -163,6 +219,12 @@ fn Counts(comptime E: type) type {return self._values.get(tag);}+ pub fn total(self: *const Self) u32 {+ var sum: u32 = 0;+ for (self._values.values) |v| sum += v;+ return sum;+ }+fn incr(self: *Self, tag: E) u32 {const ptr = self._values.getPtr(tag);ptr.* += 1;@@ -196,6 +258,25 @@ test "PageStatus: missing globals from uncaught ReferenceErrors" {try testing.expectEqual("[\"Path2D\",\"someUndefinedThing\",\"WebGL2RenderingContext\"]", json);}+test "PageStatus: failed and blocked sub-resources" {+ testing.expectLog(&.{.http});+ const page = try testing.pageTest("page_subresource_status.html", .{ .wait_until_done = true });+ defer page.close();++ const status = &page.frame().?.page.status;+ try testing.expectEqual(1, status.failed.get(.script));+ try testing.expectEqual(0, status.blocked.get(.script));++ // 403, 429 and the unreachable host; the 204 and the abort don't count.+ try testing.expectEqual(3, status.failed.get(.fetch));+ try testing.expectEqual(2, status.blocked.get(.fetch));++ try testing.expectEqual(1, status.failed.get(.xhr));+ try testing.expectEqual(0, status.blocked.get(.xhr));++ try testing.expectEqual(.vercel, status.resource_challenge.?);+}+test "PageStatus: noteException only takes well-formed ReferenceErrors" {var status: PageStatus = .{};status.noteException("TypeError: Path2D is not defined");diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zigindex cd172fd5f..60babdde3 100644--- a/src/browser/ScriptManagerBase.zig+++ b/src/browser/ScriptManagerBase.zig@@ -729,6 +729,7 @@ pub const Script = struct {const self: *Script = @ptrCast(@alignCast(transfer.req.ctx));self.status = transfer.responseStatus().?;+ self.manager.owner.pageStatus().noteResponse(.script, transfer);if (transfer.responseStatus() != 200) {log.info(.http, "script header", .{.req = transfer,@@ -841,6 +842,9 @@ pub const Script = struct {pub fn errorCallback(ctx: *anyopaque, err: anyerror) void {const self: *Script = @ptrCast(@alignCast(ctx));+ if (self.status == 0) {+ self.manager.owner.pageStatus().noteError(.script, err);+ }if (self.status == 404) {log.info(.http, "script 404", .{.req = self.url,diff --git a/src/browser/global_scope.zig b/src/browser/global_scope.zigindex dea4cd58e..4a3fe5766 100644--- a/src/browser/global_scope.zig+++ b/src/browser/global_scope.zig@@ -33,6 +33,7 @@ const WorkerGlobalScope = @import("webapi/WorkerGlobalScope.zig");const JS = @import("js/js.zig");const Frame = @import("Frame.zig");const Session = @import("Session.zig");+const PageStatus = @import("PageStatus.zig");const referrer = @import("referrer.zig");const EventManagerBase = @import("EventManagerBase.zig");@@ -175,6 +176,12 @@ pub const GlobalScope = union(enum) {};}+ pub fn pageStatus(self: GlobalScope) *PageStatus {+ return switch (self) {+ inline else => |g| &g.page.status,+ };+ }+// HttpClient.Owner of the current global (Frame or WGS). Used by code// that needs to register an in-flight network operation against the// owning scope without caring whether it's a Frame or a Worker — e.g.diff --git a/src/browser/tests/page_subresource_status.html b/src/browser/tests/page_subresource_status.htmlnew file mode 100644index 000000000..49bb76abb--- /dev/null+++ b/src/browser/tests/page_subresource_status.html@@ -0,0 +1,21 @@+<!DOCTYPE html>+<html>+<body>+ <!-- Fixture for PageStatus.failed/blocked/resource_challenge. -->+ <script src="/404.js"></script>+ <script>+ fetch('/status/403');+ fetch('/challenge/vercel');+ fetch('/status/204');+ fetch('http://127.0.0.1:1/unreachable').catch(() => {});++ const aborted = new AbortController();+ fetch('/xhr/slow', { signal: aborted.signal }).catch(() => {});+ aborted.abort();++ const xhr = new XMLHttpRequest();+ xhr.open('GET', '/xhr/500');+ xhr.send();+ </script>+</body>+</html>diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 937b64853..446aba1aa 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -1270,6 +1270,14 @@ fn weaknessNote(arena: std.mem.Allocator, frame: *const lp.Frame) !?[]const u8 {if (iframes > 0) {try notes.append(arena, try arena.print("{d} iframe(s) were not loaded: subframe loading is off.", .{iframes}));}+ if (status.resource_challenge) |challenge| {+ try notes.append(arena, try arena.print("A {s} bot challenge blocked some of the page's own requests: content they load is missing.", .{@tagName(challenge)}));+ } else {+ const blocked = status.blocked.total();+ if (blocked > 0) {+ try notes.append(arena, try arena.print("{d} of the page's own request(s) were refused with HTTP 403/429, likely bot detection or rate limiting: content they load is missing.", .{blocked}));+ }+ }if (notes.items.len == 0) {return null;}diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex ed1e3ce4a..0c6ae0e20 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -214,6 +214,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {const is_opaque = self._no_cors and transfer.client.obey_cors and transfer._cors_cross_origin;const status = transfer.responseStatus().?;+ self._exec.page.status.noteResponse(.fetch, transfer);if (is_opaque or Response.isNullBodyStatus(status) or (self._manual_redirect and HttpClient.isRedirectStatus(status))) {self._null_body = true;}@@ -349,6 +350,10 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {.err = err,});+ if (self._headers_done == false) {+ self._exec.page.status.noteError(.fetch, err);+ }+var response = self._response;response._http_transfer = null;diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zigindex 1dafbaeed..9dade46bf 100644--- a/src/browser/webapi/net/XMLHttpRequest.zig+++ b/src/browser/webapi/net/XMLHttpRequest.zig@@ -712,6 +712,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {try self.applyResponseHeaders(transfer.responseHeaders());self._response_status = transfer.responseStatus().?;+ self._exec.page.status.noteResponse(.xhr, transfer);if (transfer.statusText()) |st| {self._response_status_text = try self._arena.dupe(u8, st);}@@ -777,6 +778,9 @@ fn httpDoneCallback(ctx: *anyopaque) !void {fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));+ if (self._response_status == 0) {+ self._exec.page.status.noteError(.xhr, err);+ }// handleError can execute JS, which could .send() again: clear this now.self._http_transfer = null;self.handleError(err);diff --git a/src/server/cdp/domains/lp.zig b/src/server/cdp/domains/lp.zigindex d0a627b66..2439e0b97 100644--- a/src/server/cdp/domains/lp.zig+++ b/src/server/cdp/domains/lp.zig@@ -329,6 +329,9 @@ fn getPageStatus(cmd: anytype) !void {.unsupported = status.unsupported,.skipped = status.skipped,.missingGlobals = status.missing_globals,+ .failed = status.failed,+ .blocked = status.blocked,+ .resourceChallenge = status.resource_challenge,}, .{});}@@ -795,6 +798,10 @@ test "cdp.lp: getPageStatus" {try testing.expectEqual(1, skipped.get("image").?.integer);try testing.expectEqual(1, skipped.get("stylesheet").?.integer);+ try testing.expectEqual(0, result.get("failed").?.object.get("fetch").?.integer);+ try testing.expectEqual(0, result.get("blocked").?.object.get("fetch").?.integer);+ try testing.expectEqual(null, result.get("resourceChallenge"));+// A new top-level navigation starts over.try bc.session.initiateRootNavigation(page.frame_id, "http://localhost:9582/src/browser/tests/cdp/dialog.html", .{ .reason = .address_bar, .kind = .{ .push = null } });try testing.waitForPage(bc);