Lightpanda: the headless browser designed for AI and automation
add internal flag to Request
Muki Kiboigod891084a8b64
diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zigindex 026974bcb..35f12b867 100644--- a/src/browser/HttpClient.zig+++ b/src/browser/HttpClient.zig@@ -1324,7 +1324,10 @@ pub const Request = struct {credentials: ?[:0]const u8 = null,notification: *Notification,timeout_ms: u32 = 0,- skip_robots: bool = false,++ // Requests that are internal to the browser and skip various layers,+ // these do not need to be deferred and do not obey robots.txt.+ internal: bool = false,// When false, the caller does not guarantee that the body outlives the// transfer, and thus we'll need to dupe it.diff --git a/src/network/layer/DeferringLayer.zig b/src/network/layer/DeferringLayer.zigindex fd69b5c65..b619ee2ee 100644--- a/src/network/layer/DeferringLayer.zig+++ b/src/network/layer/DeferringLayer.zig@@ -50,6 +50,10 @@ pub fn deinit(self: *DeferringLayer) void {fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {const self: *DeferringLayer = @ptrCast(@alignCast(ptr));+ if (transfer.req.internal) {+ return self.next.request(transfer);+ }+const arena = try self.network.app.arena_pool.acquire(.small, "DeferringContext");errdefer self.network.app.arena_pool.release(arena);diff --git a/src/network/layer/RobotsLayer.zig b/src/network/layer/RobotsLayer.zigindex 6d979bfcf..7dc785739 100644--- a/src/network/layer/RobotsLayer.zig+++ b/src/network/layer/RobotsLayer.zig@@ -57,7 +57,7 @@ pub fn deinit(self: *RobotsLayer, allocator: Allocator) void {fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {const self: *RobotsLayer = @ptrCast(@alignCast(ptr));- if (transfer.req.skip_robots) {+ if (transfer.req.internal) {return self.next.request(transfer);}@@ -132,7 +132,7 @@ fn fetchRobotsThenRequest(errdefer new_req.headers.deinit();new_req.method = .GET;new_req.url = robots_url;- new_req.skip_robots = true;+ new_req.internal = true;new_req.resource_type = .fetch;new_req.body = null;new_req.ctx = robots_ctx;