Skip to content

Lightpanda: the headless browser designed for AI and automation

ops: extend PageStatus to include fetch statistics

Karl Seguinf499fbc84f66
diff --git a/src/browser/PageStatus.zig b/src/browser/PageStatus.zig
index bc6b193cf..990ed25d6 100644
--- a/src/browser/PageStatus.zig
+++ b/src/browser/PageStatus.zig
@@ -19,6 +19,9 @@
const std = @import("std");
const lp = @import("lightpanda");
+const http = @import("../network/http.zig");
+const HttpClient = @import("../network/HttpClient.zig");
+
const log = lp.log;
const PageStatus = @This();
@@ -35,9 +38,25 @@ pub const Skipped = enum {
stylesheet,
};
+// The sub-resources the page itself asks for, and which we load.
+pub const Loaded = enum {
+ script,
+ fetch,
+ xhr,
+};
+
js_errors: u32 = 0,
skipped: Counts(Skipped) = .{},
unsupported: Counts(Unsupported) = .{},
+// Sub-resource loads that got a status >= 400 or a network error. Aborts and
+// requests we refused ourselves (robots, --block-urls, adblock) aren't the
+// site failing, so they don't count.
+failed: Counts(Loaded) = .{},
+// The 403s and 429s among `failed`: a bot wall or rate limit on the page's
+// own API calls often leaves the document itself a 200.
+blocked: Counts(Loaded) = .{},
+// The first recognized bot challenge on a sub-resource.
+resource_challenge: ?HttpClient.BotChallenge = null,
// Globals that uncaught ReferenceErrors named. Most of what a page needs and
// we lack shows up here first, before any call we could count: TradingView
// dies at `new Path2D()` and never reaches getContext. Deduped and capped;
@@ -107,6 +126,43 @@ pub fn noteSkipped(self: *PageStatus, what: Skipped) void {
}
}
+// A sub-resource's final response headers.
+pub fn noteResponse(self: *PageStatus, what: Loaded, transfer: *const HttpClient.Transfer) void {
+ const status = transfer.responseStatus() orelse return;
+
+ // Before the status check: AWS WAF challenges with a 202.
+ if (self.resource_challenge == null) {
+ if (transfer.botChallenge()) |challenge| {
+ self.resource_challenge = challenge;
+ log.warn(.http, "sub-resource bot challenge", .{ .challenge = challenge, .resource = what, .url = transfer.req.url });
+ }
+ }
+
+ if (status < 400) {
+ return;
+ }
+ _ = self.failed.incr(what);
+
+ if (status == 403 or status == 429) {
+ if (self.blocked.incr(what) == 1) {
+ log.info(.http, "sub-resource blocked", .{ .resource = what, .status = status, .url = transfer.req.url });
+ }
+ }
+}
+
+// A sub-resource load that ended without a usable response. Callers only
+// report errors that came before any response headers, which noteResponse
+// has already counted.
+pub fn noteError(self: *PageStatus, what: Loaded, err: anyerror) void {
+ if (err == error.UrlBlocked) {
+ return;
+ }
+ switch (http.errorReason(err)) {
+ .aborted, .robots_blocked => return,
+ else => _ = self.failed.incr(what),
+ }
+}
+
pub const MissingGlobals = struct {
const max = 8;
const max_len = 48;
@@ -163,6 +219,12 @@ fn Counts(comptime E: type) type {
return self._values.get(tag);
}
+ pub fn total(self: *const Self) u32 {
+ var sum: u32 = 0;
+ for (self._values.values) |v| sum += v;
+ return sum;
+ }
+
fn incr(self: *Self, tag: E) u32 {
const ptr = self._values.getPtr(tag);
ptr.* += 1;
@@ -196,6 +258,25 @@ test "PageStatus: missing globals from uncaught ReferenceErrors" {
try testing.expectEqual("[\"Path2D\",\"someUndefinedThing\",\"WebGL2RenderingContext\"]", json);
}
+test "PageStatus: failed and blocked sub-resources" {
+ testing.expectLog(&.{.http});
+ const page = try testing.pageTest("page_subresource_status.html", .{ .wait_until_done = true });
+ defer page.close();
+
+ const status = &page.frame().?.page.status;
+ try testing.expectEqual(1, status.failed.get(.script));
+ try testing.expectEqual(0, status.blocked.get(.script));
+
+ // 403, 429 and the unreachable host; the 204 and the abort don't count.
+ try testing.expectEqual(3, status.failed.get(.fetch));
+ try testing.expectEqual(2, status.blocked.get(.fetch));
+
+ try testing.expectEqual(1, status.failed.get(.xhr));
+ try testing.expectEqual(0, status.blocked.get(.xhr));
+
+ try testing.expectEqual(.vercel, status.resource_challenge.?);
+}
+
test "PageStatus: noteException only takes well-formed ReferenceErrors" {
var status: PageStatus = .{};
status.noteException("TypeError: Path2D is not defined");
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zig
index cd172fd5f..60babdde3 100644
--- a/src/browser/ScriptManagerBase.zig
+++ b/src/browser/ScriptManagerBase.zig
@@ -729,6 +729,7 @@ pub const Script = struct {
const self: *Script = @ptrCast(@alignCast(transfer.req.ctx));
self.status = transfer.responseStatus().?;
+ self.manager.owner.pageStatus().noteResponse(.script, transfer);
if (transfer.responseStatus() != 200) {
log.info(.http, "script header", .{
.req = transfer,
@@ -841,6 +842,9 @@ pub const Script = struct {
pub fn errorCallback(ctx: *anyopaque, err: anyerror) void {
const self: *Script = @ptrCast(@alignCast(ctx));
+ if (self.status == 0) {
+ self.manager.owner.pageStatus().noteError(.script, err);
+ }
if (self.status == 404) {
log.info(.http, "script 404", .{
.req = self.url,
diff --git a/src/browser/global_scope.zig b/src/browser/global_scope.zig
index dea4cd58e..4a3fe5766 100644
--- a/src/browser/global_scope.zig
+++ b/src/browser/global_scope.zig
@@ -33,6 +33,7 @@ const WorkerGlobalScope = @import("webapi/WorkerGlobalScope.zig");
const JS = @import("js/js.zig");
const Frame = @import("Frame.zig");
const Session = @import("Session.zig");
+const PageStatus = @import("PageStatus.zig");
const referrer = @import("referrer.zig");
const EventManagerBase = @import("EventManagerBase.zig");
@@ -175,6 +176,12 @@ pub const GlobalScope = union(enum) {
};
}
+ pub fn pageStatus(self: GlobalScope) *PageStatus {
+ return switch (self) {
+ inline else => |g| &g.page.status,
+ };
+ }
+
// HttpClient.Owner of the current global (Frame or WGS). Used by code
// that needs to register an in-flight network operation against the
// owning scope without caring whether it's a Frame or a Worker — e.g.
diff --git a/src/browser/tests/page_subresource_status.html b/src/browser/tests/page_subresource_status.html
new file mode 100644
index 000000000..49bb76abb
--- /dev/null
+++ b/src/browser/tests/page_subresource_status.html
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<body>
+ <!-- Fixture for PageStatus.failed/blocked/resource_challenge. -->
+ <script src="/404.js"></script>
+ <script>
+ fetch('/status/403');
+ fetch('/challenge/vercel');
+ fetch('/status/204');
+ fetch('http://127.0.0.1:1/unreachable').catch(() => {});
+
+ const aborted = new AbortController();
+ fetch('/xhr/slow', { signal: aborted.signal }).catch(() => {});
+ aborted.abort();
+
+ const xhr = new XMLHttpRequest();
+ xhr.open('GET', '/xhr/500');
+ xhr.send();
+ </script>
+</body>
+</html>
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 937b64853..446aba1aa 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -1270,6 +1270,14 @@ fn weaknessNote(arena: std.mem.Allocator, frame: *const lp.Frame) !?[]const u8 {
if (iframes > 0) {
try notes.append(arena, try arena.print("{d} iframe(s) were not loaded: subframe loading is off.", .{iframes}));
}
+ if (status.resource_challenge) |challenge| {
+ try notes.append(arena, try arena.print("A {s} bot challenge blocked some of the page's own requests: content they load is missing.", .{@tagName(challenge)}));
+ } else {
+ const blocked = status.blocked.total();
+ if (blocked > 0) {
+ try notes.append(arena, try arena.print("{d} of the page's own request(s) were refused with HTTP 403/429, likely bot detection or rate limiting: content they load is missing.", .{blocked}));
+ }
+ }
if (notes.items.len == 0) {
return null;
}
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index ed1e3ce4a..0c6ae0e20 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -214,6 +214,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
const is_opaque = self._no_cors and transfer.client.obey_cors and transfer._cors_cross_origin;
const status = transfer.responseStatus().?;
+ self._exec.page.status.noteResponse(.fetch, transfer);
if (is_opaque or Response.isNullBodyStatus(status) or (self._manual_redirect and HttpClient.isRedirectStatus(status))) {
self._null_body = true;
}
@@ -349,6 +350,10 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
.err = err,
});
+ if (self._headers_done == false) {
+ self._exec.page.status.noteError(.fetch, err);
+ }
+
var response = self._response;
response._http_transfer = null;
diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig
index 1dafbaeed..9dade46bf 100644
--- a/src/browser/webapi/net/XMLHttpRequest.zig
+++ b/src/browser/webapi/net/XMLHttpRequest.zig
@@ -712,6 +712,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
try self.applyResponseHeaders(transfer.responseHeaders());
self._response_status = transfer.responseStatus().?;
+ self._exec.page.status.noteResponse(.xhr, transfer);
if (transfer.statusText()) |st| {
self._response_status_text = try self._arena.dupe(u8, st);
}
@@ -777,6 +778,9 @@ fn httpDoneCallback(ctx: *anyopaque) !void {
fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
+ if (self._response_status == 0) {
+ self._exec.page.status.noteError(.xhr, err);
+ }
// handleError can execute JS, which could .send() again: clear this now.
self._http_transfer = null;
self.handleError(err);
diff --git a/src/server/cdp/domains/lp.zig b/src/server/cdp/domains/lp.zig
index d0a627b66..2439e0b97 100644
--- a/src/server/cdp/domains/lp.zig
+++ b/src/server/cdp/domains/lp.zig
@@ -329,6 +329,9 @@ fn getPageStatus(cmd: anytype) !void {
.unsupported = status.unsupported,
.skipped = status.skipped,
.missingGlobals = status.missing_globals,
+ .failed = status.failed,
+ .blocked = status.blocked,
+ .resourceChallenge = status.resource_challenge,
}, .{});
}
@@ -795,6 +798,10 @@ test "cdp.lp: getPageStatus" {
try testing.expectEqual(1, skipped.get("image").?.integer);
try testing.expectEqual(1, skipped.get("stylesheet").?.integer);
+ try testing.expectEqual(0, result.get("failed").?.object.get("fetch").?.integer);
+ try testing.expectEqual(0, result.get("blocked").?.object.get("fetch").?.integer);
+ try testing.expectEqual(null, result.get("resourceChallenge"));
+
// A new top-level navigation starts over.
try bc.session.initiateRootNavigation(page.frame_id, "http://localhost:9582/src/browser/tests/cdp/dialog.html", .{ .reason = .address_bar, .kind = .{ .push = null } });
try testing.waitForPage(bc);