Skip to content

Lightpanda: the headless browser designed for AI and automation

use robots_url before releasing arena in all RobotGate paths

Muki Kiboigob55b8966fd0d
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index ff1e7f06c..07851dac8 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -196,8 +196,7 @@ const RobotsContext = struct {
200 => {
if (self.buffer.items.len == 0) {
// Empty robots.txt means we can short-circuit the allowed path.
- try network.robot_store.putAllowed(robots_url);
- self.resolve(.{ .decision = .allowed });
+ self.settle(.{ .decision = .allowed });
return;
}
@@ -209,14 +208,12 @@ const RobotsContext = struct {
// Our parser does already leniently handle malformed input and takes whichever rules it can parse.
// On this case of an allocation failure, it is our fault so we put it as disallowed.
log.warn(.browser, "error while parsing robots.txt", .{ .robots_url = robots_url, .err = err });
- try network.robot_store.putDisallowed(robots_url);
- self.resolve(.{ .decision = .blocked });
+ self.settle(.{ .decision = .blocked });
return;
};
- self.resolve(.{ .robots = robots });
// BE CAREFUL: robots can be invalidated after this call
- try network.robot_store.put(robots_url, robots);
+ self.settle(.{ .robots = robots });
},
// Unauthorized/Forbidden: treat as fully disallowed since we can't verify permissions.
401, 403 => {
@@ -224,14 +221,12 @@ const RobotsContext = struct {
.url = robots_url,
.status = self.status,
});
- try network.robot_store.putDisallowed(robots_url);
- self.resolve(.{ .decision = .blocked });
+ self.settle(.{ .decision = .blocked });
},
// RFC9309: Unavailable (400-499) means that we may access any resources on the server.
400, 402, 404...499 => {
log.debug(.http, "robots.txt unavailable", .{ .url = robots_url });
- try network.robot_store.putAllowed(robots_url);
- self.resolve(.{ .decision = .allowed });
+ self.settle(.{ .decision = .allowed });
},
// RFC9309: Unreachable (500-599) means that we are completely disallowed.
500...599 => {
@@ -239,16 +234,14 @@ const RobotsContext = struct {
.url = robots_url,
.status = self.status,
});
- try network.robot_store.putDisallowed(robots_url);
- self.resolve(.{ .decision = .blocked });
+ self.settle(.{ .decision = .blocked });
},
else => {
log.debug(.http, "unexpected status on robots", .{
.url = robots_url,
.status = self.status,
});
- try network.robot_store.putDisallowed(robots_url);
- self.resolve(.{ .decision = .blocked });
+ self.settle(.{ .decision = .blocked });
},
}
}
@@ -257,7 +250,7 @@ const RobotsContext = struct {
const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
log.warn(.http, "robots fetch failed", .{ .err = err });
- self.resolve(.{ .decision = .allowed });
+ self.settle(.{ .decision = .allowed });
}
fn shutdownCallback(ctx_ptr: *anyopaque) void {
@@ -270,10 +263,27 @@ const RobotsContext = struct {
arena.release();
}
- fn resolve(self: *RobotsContext, outcome: Outcome) void {
- const gate = self.gate;
+ fn settle(self: *RobotsContext, outcome: RobotsGate.Outcome) void {
const arena = self.arena;
+ defer arena.release();
+
+ const gate = self.gate;
+ const network = gate.network;
+
gate.flushPending(self.robots_url, outcome);
- arena.release();
+
+ switch (outcome) {
+ .decision => |d| switch (d) {
+ .allowed => network.robot_store.putAllowed(self.robots_url) catch |err| {
+ log.warn(.browser, "failed to cache robots decision", .{ .url = self.robots_url, .err = err });
+ },
+ .blocked => network.robot_store.putDisallowed(self.robots_url) catch |err| {
+ log.warn(.browser, "failed to cache robots decision", .{ .url = self.robots_url, .err = err });
+ },
+ },
+ .robots => |r| network.robot_store.put(self.robots_url, r) catch |err| {
+ log.warn(.browser, "failed to cache robots rules", .{ .url = self.robots_url, .err = err });
+ },
+ }
}
};