Lightpanda: the headless browser designed for AI and automation
use robots_url before releasing arena in all RobotGate paths
Muki Kiboigob55b8966fd0d
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex ff1e7f06c..07851dac8 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -196,8 +196,7 @@ const RobotsContext = struct {200 => {if (self.buffer.items.len == 0) {// Empty robots.txt means we can short-circuit the allowed path.- try network.robot_store.putAllowed(robots_url);- self.resolve(.{ .decision = .allowed });+ self.settle(.{ .decision = .allowed });return;}@@ -209,14 +208,12 @@ const RobotsContext = struct {// Our parser does already leniently handle malformed input and takes whichever rules it can parse.// On this case of an allocation failure, it is our fault so we put it as disallowed.log.warn(.browser, "error while parsing robots.txt", .{ .robots_url = robots_url, .err = err });- try network.robot_store.putDisallowed(robots_url);- self.resolve(.{ .decision = .blocked });+ self.settle(.{ .decision = .blocked });return;};- self.resolve(.{ .robots = robots });// BE CAREFUL: robots can be invalidated after this call- try network.robot_store.put(robots_url, robots);+ self.settle(.{ .robots = robots });},// Unauthorized/Forbidden: treat as fully disallowed since we can't verify permissions.401, 403 => {@@ -224,14 +221,12 @@ const RobotsContext = struct {.url = robots_url,.status = self.status,});- try network.robot_store.putDisallowed(robots_url);- self.resolve(.{ .decision = .blocked });+ self.settle(.{ .decision = .blocked });},// RFC9309: Unavailable (400-499) means that we may access any resources on the server.400, 402, 404...499 => {log.debug(.http, "robots.txt unavailable", .{ .url = robots_url });- try network.robot_store.putAllowed(robots_url);- self.resolve(.{ .decision = .allowed });+ self.settle(.{ .decision = .allowed });},// RFC9309: Unreachable (500-599) means that we are completely disallowed.500...599 => {@@ -239,16 +234,14 @@ const RobotsContext = struct {.url = robots_url,.status = self.status,});- try network.robot_store.putDisallowed(robots_url);- self.resolve(.{ .decision = .blocked });+ self.settle(.{ .decision = .blocked });},else => {log.debug(.http, "unexpected status on robots", .{.url = robots_url,.status = self.status,});- try network.robot_store.putDisallowed(robots_url);- self.resolve(.{ .decision = .blocked });+ self.settle(.{ .decision = .blocked });},}}@@ -257,7 +250,7 @@ const RobotsContext = struct {const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));log.warn(.http, "robots fetch failed", .{ .err = err });- self.resolve(.{ .decision = .allowed });+ self.settle(.{ .decision = .allowed });}fn shutdownCallback(ctx_ptr: *anyopaque) void {@@ -270,10 +263,27 @@ const RobotsContext = struct {arena.release();}- fn resolve(self: *RobotsContext, outcome: Outcome) void {- const gate = self.gate;+ fn settle(self: *RobotsContext, outcome: RobotsGate.Outcome) void {const arena = self.arena;+ defer arena.release();++ const gate = self.gate;+ const network = gate.network;+gate.flushPending(self.robots_url, outcome);- arena.release();++ switch (outcome) {+ .decision => |d| switch (d) {+ .allowed => network.robot_store.putAllowed(self.robots_url) catch |err| {+ log.warn(.browser, "failed to cache robots decision", .{ .url = self.robots_url, .err = err });+ },+ .blocked => network.robot_store.putDisallowed(self.robots_url) catch |err| {+ log.warn(.browser, "failed to cache robots decision", .{ .url = self.robots_url, .err = err });+ },+ },+ .robots => |r| network.robot_store.put(self.robots_url, r) catch |err| {+ log.warn(.browser, "failed to cache robots rules", .{ .url = self.robots_url, .err = err });+ },+ }}};