Skip to content

Lightpanda: the headless browser designed for AI and automation

add internal flag to Request

Muki Kiboigod891084a8b64
diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig
index 026974bcb..35f12b867 100644
--- a/src/browser/HttpClient.zig
+++ b/src/browser/HttpClient.zig
@@ -1324,7 +1324,10 @@ pub const Request = struct {
credentials: ?[:0]const u8 = null,
notification: *Notification,
timeout_ms: u32 = 0,
- skip_robots: bool = false,
+
+ // Requests that are internal to the browser and skip various layers,
+ // these do not need to be deferred and do not obey robots.txt.
+ internal: bool = false,
// When false, the caller does not guarantee that the body outlives the
// transfer, and thus we'll need to dupe it.
diff --git a/src/network/layer/DeferringLayer.zig b/src/network/layer/DeferringLayer.zig
index fd69b5c65..b619ee2ee 100644
--- a/src/network/layer/DeferringLayer.zig
+++ b/src/network/layer/DeferringLayer.zig
@@ -50,6 +50,10 @@ pub fn deinit(self: *DeferringLayer) void {
fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
const self: *DeferringLayer = @ptrCast(@alignCast(ptr));
+ if (transfer.req.internal) {
+ return self.next.request(transfer);
+ }
+
const arena = try self.network.app.arena_pool.acquire(.small, "DeferringContext");
errdefer self.network.app.arena_pool.release(arena);
diff --git a/src/network/layer/RobotsLayer.zig b/src/network/layer/RobotsLayer.zig
index 6d979bfcf..7dc785739 100644
--- a/src/network/layer/RobotsLayer.zig
+++ b/src/network/layer/RobotsLayer.zig
@@ -57,7 +57,7 @@ pub fn deinit(self: *RobotsLayer, allocator: Allocator) void {
fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
const self: *RobotsLayer = @ptrCast(@alignCast(ptr));
- if (transfer.req.skip_robots) {
+ if (transfer.req.internal) {
return self.next.request(transfer);
}
@@ -132,7 +132,7 @@ fn fetchRobotsThenRequest(
errdefer new_req.headers.deinit();
new_req.method = .GET;
new_req.url = robots_url;
- new_req.skip_robots = true;
+ new_req.internal = true;
new_req.resource_type = .fetch;
new_req.body = null;
new_req.ctx = robots_ctx;