Skip to content

Lightpanda: the headless browser designed for AI and automation

Surface server-side agent-discovery signals (Link header + Content-Signal)

Navid EMADb0a087450dee
diff --git a/src/browser/structured_data.zig b/src/browser/structured_data.zig
index 09cf72a2a..21e063b86 100644
--- a/src/browser/structured_data.zig
+++ b/src/browser/structured_data.zig
@@ -39,6 +39,13 @@ pub const AlternateLink = struct {
title: ?[]const u8,
};
+/// A relation discovered in the HTTP `Link:` response header (RFC 8288),
+/// restricted to the registered relations an agent can act on.
+pub const LinkRel = struct {
+ rel: []const u8,
+ href: []const u8,
+};
+
pub const StructuredData = struct {
json_ld: []const []const u8,
open_graph: []const Property,
@@ -46,6 +53,7 @@ pub const StructuredData = struct {
meta: []const Property,
links: []const Property,
alternate: []const AlternateLink,
+ link_headers: []const LinkRel,
pub fn jsonStringify(self: *const StructuredData, jw: anytype) !void {
try jw.beginObject();
@@ -89,6 +97,23 @@ pub const StructuredData = struct {
try jw.endArray();
}
+ // Emitted only when present so existing consumers of the CDP/MCP
+ // structured-data shape see no new key on sites without a `Link:`
+ // response header.
+ if (self.link_headers.len > 0) {
+ try jw.objectField("linkHeaders");
+ try jw.beginArray();
+ for (self.link_headers) |lh| {
+ try jw.beginObject();
+ try jw.objectField("rel");
+ try jw.write(lh.rel);
+ try jw.objectField("href");
+ try jw.write(lh.href);
+ try jw.endObject();
+ }
+ try jw.endArray();
+ }
+
try jw.endObject();
}
};
@@ -144,6 +169,7 @@ pub fn collectStructuredData(
var meta: std.ArrayList(Property) = .empty;
var links: std.ArrayList(Property) = .empty;
var alternate: std.ArrayList(AlternateLink) = .empty;
+ var link_headers: std.ArrayList(LinkRel) = .empty;
// Extract language from the root <html> element.
if (root.is(Element)) |root_el| {
@@ -182,6 +208,10 @@ pub fn collectStructuredData(
}
}
+ // The `Link:` response header lives on the frame, not in the DOM, so it is
+ // collected separately from the navigation's retained headers.
+ try collectLinkHeaders(arena, frame, &link_headers);
+
return .{
.json_ld = json_ld.items,
.open_graph = open_graph.items,
@@ -189,9 +219,133 @@ pub fn collectStructuredData(
.meta = meta.items,
.links = links.items,
.alternate = alternate.items,
+ .link_headers = link_headers.items,
};
}
+/// Registered link relations we surface from the HTTP `Link:` response header
+/// (RFC 8288). `service-doc`/`service-desc` come from RFC 8631; `api` is in the
+/// IANA Link Relations registry. These let an agent discover a site's developer
+/// docs / API description without scraping.
+const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" };
+
+/// Parse every `Link:` response header retained on the frame, surfacing only the
+/// registered, agent-useful relations. Relative targets are resolved against the
+/// frame's base URL, mirroring `collectLink`.
+fn collectLinkHeaders(
+ arena: Allocator,
+ frame: *Frame,
+ out: *std.ArrayList(LinkRel),
+) !void {
+ for (frame._http_headers.items) |header| {
+ if (!std.ascii.eqlIgnoreCase(header.name, "link")) continue;
+
+ var it: LinkHeaderIterator = .{ .value = header.value };
+ while (it.next()) |entry| {
+ const rel_value = entry.rel orelse continue;
+
+ // The `rel` parameter is a space-separated list of relation types.
+ var rel_it = std.mem.tokenizeAny(u8, rel_value, " \t");
+ while (rel_it.next()) |rel_token| {
+ const known = for (header_link_rels) |candidate| {
+ if (std.ascii.eqlIgnoreCase(rel_token, candidate)) break candidate;
+ } else continue;
+
+ const href = URL.resolve(arena, frame.base(), entry.target, .{ .encoding = frame.charset }) catch entry.target;
+
+ // Drop duplicate (rel, href) pairs — a target can legitimately
+ // carry the same relation across repeated headers.
+ const dup = for (out.items) |existing| {
+ if (std.mem.eql(u8, existing.rel, known) and std.mem.eql(u8, existing.href, href)) break true;
+ } else false;
+ if (!dup) try out.append(arena, .{ .rel = known, .href = href });
+ }
+ }
+ }
+}
+
+/// One `< target >; param=value; ...` entry of an RFC 8288 `Link` header value.
+const LinkHeaderEntry = struct {
+ target: []const u8,
+ rel: ?[]const u8,
+};
+
+/// Splits a `Link:` header field value into its comma-separated link-values,
+/// skipping commas that fall inside the `<...>` target or a `"..."` quoted
+/// parameter value (RFC 8288 §3).
+const LinkHeaderIterator = struct {
+ value: []const u8,
+ i: usize = 0,
+
+ fn next(self: *LinkHeaderIterator) ?LinkHeaderEntry {
+ const v = self.value;
+
+ // Loop (rather than recurse) over malformed segments so stack usage is
+ // independent of the attacker-controlled header length.
+ while (true) {
+ // Skip separators and whitespace between link-values. The comma that
+ // ended the previous link-value is consumed here.
+ while (self.i < v.len and (std.ascii.isWhitespace(v[self.i]) or v[self.i] == ',')) : (self.i += 1) {}
+ if (self.i >= v.len) return null;
+
+ // A link-value must begin with the angle-bracketed target.
+ if (v[self.i] != '<') {
+ self.skipToNextComma();
+ continue;
+ }
+ const target_start = self.i + 1;
+ const gt = std.mem.indexOfScalarPos(u8, v, target_start, '>') orelse {
+ self.i = v.len;
+ return null;
+ };
+ const target = v[target_start..gt];
+ self.i = gt + 1;
+
+ // Parameters run to the next top-level comma, left at `self.i`.
+ const params_start = self.i;
+ self.skipToNextComma();
+ return .{ .target = target, .rel = extractRel(v[params_start..self.i]) };
+ }
+ }
+
+ // Advance until `self.i` sits on the next top-level comma (one outside a
+ // `"..."` quoted-string, honouring RFC 7230 backslash escapes) or the end
+ // of the value. The comma itself is left for the leading-separator skip.
+ fn skipToNextComma(self: *LinkHeaderIterator) void {
+ const v = self.value;
+ var in_quotes = false;
+ while (self.i < v.len) : (self.i += 1) {
+ const c = v[self.i];
+ if (c == '\\' and in_quotes) {
+ // Skip the escaped byte; guard a trailing backslash.
+ if (self.i + 1 < v.len) self.i += 1;
+ } else if (c == '"') {
+ in_quotes = !in_quotes;
+ } else if (c == ',' and !in_quotes) {
+ return;
+ }
+ }
+ }
+};
+
+/// Return the (unquoted) value of the `rel` parameter from a link-value's
+/// parameter list, or null when absent.
+fn extractRel(params: []const u8) ?[]const u8 {
+ var it = std.mem.splitScalar(u8, params, ';');
+ while (it.next()) |raw_param| {
+ const param = std.mem.trim(u8, raw_param, &std.ascii.whitespace);
+ const eq = std.mem.indexOfScalar(u8, param, '=') orelse continue;
+ if (!std.ascii.eqlIgnoreCase(std.mem.trim(u8, param[0..eq], &std.ascii.whitespace), "rel")) continue;
+
+ var val = std.mem.trim(u8, param[eq + 1 ..], &std.ascii.whitespace);
+ if (val.len >= 2 and val[0] == '"' and val[val.len - 1] == '"') {
+ val = val[1 .. val.len - 1];
+ }
+ return val;
+ }
+ return null;
+}
+
fn collectJsonLd(
el: *Element,
arena: Allocator,
@@ -484,6 +638,78 @@ test "structured_data: charset and http-equiv" {
try testing.expectEqual("text/html; charset=utf-8", findProperty(data.meta, "Content-Type").?);
}
+test "structured_data: parseLinkHeader - single link with quoted rel" {
+ var it: LinkHeaderIterator = .{ .value =
+ \\<https://api.example.com/openapi.json>; rel="service-desc"
+ };
+ const entry = it.next().?;
+ try testing.expectString("https://api.example.com/openapi.json", entry.target);
+ try testing.expectString("service-desc", entry.rel.?);
+ try testing.expectEqual(null, it.next());
+}
+
+test "structured_data: parseLinkHeader - multiple links and unquoted rel" {
+ var it: LinkHeaderIterator = .{ .value =
+ \\<https://docs.example.com/>; rel=service-doc, </style.css>; rel="stylesheet"; type="text/css"
+ };
+ const first = it.next().?;
+ try testing.expectString("https://docs.example.com/", first.target);
+ try testing.expectString("service-doc", first.rel.?);
+
+ const second = it.next().?;
+ try testing.expectString("/style.css", second.target);
+ // A comma inside the quoted type param must not split the link-value.
+ try testing.expectString("stylesheet", second.rel.?);
+
+ try testing.expectEqual(null, it.next());
+}
+
+test "structured_data: parseLinkHeader - escaped quote in param does not split link" {
+ // A backslash-escaped quote inside an earlier param must not terminate the
+ // quoted-string, so the comma inside it is not treated as a separator.
+ var it: LinkHeaderIterator = .{ .value =
+ \\<https://a/>; title="x\",y"; rel="service-doc"
+ };
+ const entry = it.next().?;
+ try testing.expectString("https://a/", entry.target);
+ try testing.expectString("service-doc", entry.rel.?);
+ try testing.expectEqual(null, it.next());
+}
+
+test "structured_data: link headers from response" {
+ const frame = try testing.test_session.createPage();
+ defer testing.test_session.removePage();
+
+ // Stand in for what frameHeaderDoneCallback records from the navigation.
+ try frame._http_headers.append(frame.arena, .{ .name = "Link", .value =
+ \\<https://docs.example.com/>; rel="service-doc"
+ });
+ try frame._http_headers.append(frame.arena, .{ .name = "link", .value =
+ \\<https://api.example.com/spec>; rel="service-desc", </css/site.css>; rel="stylesheet"
+ });
+
+ const doc = frame.window._document;
+ const div = try doc.createElement("div", null, frame);
+ try frame.parseHtmlAsChildren(div.asNode(), "<title>Example</title>");
+
+ const data = try collectStructuredData(div.asNode(), frame.call_arena, frame);
+
+ // service-doc + service-desc are surfaced; stylesheet is filtered out.
+ try testing.expectEqual(2, data.link_headers.len);
+ try testing.expectString("service-doc", data.link_headers[0].rel);
+ try testing.expectString("https://docs.example.com/", data.link_headers[0].href);
+ try testing.expectString("service-desc", data.link_headers[1].rel);
+ try testing.expectString("https://api.example.com/spec", data.link_headers[1].href);
+}
+
+test "structured_data: no link header yields empty list" {
+ const data = try testStructuredData(
+ \\<link rel="canonical" href="https://example.com">
+ );
+ defer testing.test_session.removePage();
+ try testing.expectEqual(0, data.link_headers.len);
+}
+
test "structured_data: mixed content" {
const data = try testStructuredData(
\\<title>My Site</title>
diff --git a/src/cdp/domains/lp.zig b/src/cdp/domains/lp.zig
index b3b3aec0a..dd8fdd991 100644
--- a/src/cdp/domains/lp.zig
+++ b/src/cdp/domains/lp.zig
@@ -23,6 +23,7 @@ const CDP = @import("../CDP.zig");
const Node = @import("../Node.zig");
const DOMNode = @import("../../browser/webapi/Node.zig");
+const Robots = @import("../../network/Robots.zig");
const markdown = lp.markdown;
const SemanticTree = lp.SemanticTree;
@@ -36,6 +37,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
getInteractiveElements,
getNodeDetails,
getStructuredData,
+ getContentSignal,
detectForms,
clickNode,
fillNode,
@@ -51,6 +53,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
.getInteractiveElements => return getInteractiveElements(cmd),
.getNodeDetails => return getNodeDetails(cmd),
.getStructuredData => return getStructuredData(cmd),
+ .getContentSignal => return getContentSignal(cmd),
.detectForms => return detectForms(cmd),
.clickNode => return clickNode(cmd),
.fillNode => return fillNode(cmd),
@@ -201,6 +204,27 @@ fn getStructuredData(cmd: anytype) !void {
}, .{});
}
+// Advisory `Content-Signal` robots.txt preferences for the current document's
+// host (https://contentsignals.org). `available` is false when no robots.txt
+// has been fetched for the host — note this is the case unless `obey_robots`
+// is enabled, since the robots layer is what populates the store.
+fn getContentSignal(cmd: anytype) !void {
+ const bc = cmd.browser_context orelse return error.NoBrowserContext;
+ const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded;
+ const network = bc.cdp.browser.http_client.network;
+
+ const empty: []const Robots.ContentSignal = &.{};
+ const robots_url = lp.URL.getRobotsUrl(cmd.arena, frame.url) catch {
+ return cmd.sendResult(.{ .available = false, .contentSignals = empty }, .{});
+ };
+
+ const signals = network.robot_store.getContentSignals(robots_url);
+ return cmd.sendResult(.{
+ .available = signals != null,
+ .contentSignals = signals orelse empty,
+ }, .{});
+}
+
fn detectForms(cmd: anytype) !void {
const bc = cmd.browser_context orelse return error.NoBrowserContext;
const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded;
@@ -410,6 +434,25 @@ test "cdp.lp: getStructuredData" {
try testing.expect(result.get("structuredData") != null);
}
+test "cdp.lp: getContentSignal" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{});
+ _ = try bc.session.createPage();
+
+ try ctx.processMessage(.{
+ .id = 1,
+ .method = "LP.getContentSignal",
+ });
+
+ // No robots.txt fetched for the page, so the directive is reported absent
+ // rather than erroring — the command is still wired and returns the shape.
+ const result = (try ctx.getSentMessage(0)).?.object.get("result").?.object;
+ try testing.expectEqual(false, result.get("available").?.bool);
+ try testing.expect(result.get("contentSignals") != null);
+}
+
test "cdp.lp: action tools" {
var ctx = try testing.context();
defer ctx.deinit();
diff --git a/src/network/Robots.zig b/src/network/Robots.zig
index 202e0a5c7..423a308cf 100644
--- a/src/network/Robots.zig
+++ b/src/network/Robots.zig
@@ -72,11 +72,20 @@ pub const Key = enum {
@"user-agent",
allow,
disallow,
+ @"content-signal",
+};
+
+/// A declared content-usage preference from a `Content-Signal:` robots.txt
+/// directive (https://contentsignals.org), e.g. name="ai-train" value="no".
+/// Advisory metadata surfaced to the agent — it never affects crawl gating.
+pub const ContentSignal = struct {
+ name: []const u8,
+ value: []const u8,
};
/// https://www.rfc-editor.org/rfc/rfc9309.html
pub const Robots = @This();
-pub const empty: Robots = .{ .rules = &.{} };
+pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };
pub const RobotStore = struct {
const RobotsEntry = union(enum) {
@@ -156,6 +165,21 @@ pub const RobotStore = struct {
try self.map.put(self.allocator, duped, .{ .present = robots });
}
+ /// Advisory `Content-Signal` preferences declared for `url`'s host, or null
+ /// when no robots.txt has been fetched/stored for it. The returned slice is
+ /// owned by the store; callers must read it synchronously (and dupe to keep
+ /// it) and must never free it.
+ pub fn getContentSignals(self: *RobotStore, url: []const u8) ?[]const ContentSignal {
+ self.mutex.lock();
+ defer self.mutex.unlock();
+
+ const entry = self.map.get(url) orelse return null;
+ return switch (entry) {
+ .present => |robots| robots.content_signals,
+ .absent => null,
+ };
+ }
+
pub fn putAbsent(self: *RobotStore, url: []const u8) !void {
self.mutex.lock();
defer self.mutex.unlock();
@@ -166,6 +190,15 @@ pub const RobotStore = struct {
};
rules: []const Rule,
+content_signals: []const ContentSignal = &.{},
+
+/// Result of parsing a robots.txt for one user-agent: the applicable crawl
+/// rules plus any advisory `Content-Signal` preferences. The two are kept
+/// distinct so content signals can never reach `isAllowed`.
+const ParseResult = struct {
+ rules: []Rule,
+ content_signals: []ContentSignal,
+};
const State = struct {
entry: enum {
@@ -186,17 +219,56 @@ fn freeRulesInList(allocator: std.mem.Allocator, rules: []const Rule) void {
}
}
+fn freeContentSignals(allocator: std.mem.Allocator, signals: []const ContentSignal) void {
+ for (signals) |signal| {
+ allocator.free(signal.name);
+ allocator.free(signal.value);
+ }
+}
+
+/// Parse a `Content-Signal:` value — a comma-separated list of `name=value`
+/// pairs (e.g. `ai-train=no, search=yes`) — into the given list. Names and
+/// values are lower-cased and owned by `allocator`.
+fn appendContentSignals(
+ allocator: std.mem.Allocator,
+ list: *std.ArrayList(ContentSignal),
+ value: []const u8,
+) !void {
+ var iter = std.mem.splitScalar(u8, value, ',');
+ while (iter.next()) |raw_pair| {
+ const pair = std.mem.trim(u8, raw_pair, &std.ascii.whitespace);
+ const eq = std.mem.indexOfScalar(u8, pair, '=') orelse continue;
+
+ const name_raw = std.mem.trim(u8, pair[0..eq], &std.ascii.whitespace);
+ const value_raw = std.mem.trim(u8, pair[eq + 1 ..], &std.ascii.whitespace);
+ if (name_raw.len == 0) continue;
+
+ const name = try std.ascii.allocLowerString(allocator, name_raw);
+ errdefer allocator.free(name);
+ const val = try std.ascii.allocLowerString(allocator, value_raw);
+ errdefer allocator.free(val);
+
+ try list.append(allocator, .{ .name = name, .value = val });
+ }
+}
+
fn parseRulesWithUserAgent(
allocator: std.mem.Allocator,
user_agent: []const u8,
raw_bytes: []const u8,
-) ![]Rule {
+) !ParseResult {
var rules: std.ArrayList(Rule) = .empty;
defer rules.deinit(allocator);
var wildcard_rules: std.ArrayList(Rule) = .empty;
defer wildcard_rules.deinit(allocator);
+ var content_signals: std.ArrayList(ContentSignal) = .empty;
+ defer content_signals.deinit(allocator);
+
+ var wildcard_content_signals: std.ArrayList(ContentSignal) = .empty;
+ defer wildcard_content_signals.deinit(allocator);
+
var state: State = .{ .entry = .not_in_entry, .has_rules = false };
// https://en.wikipedia.org/wiki/Byte_order_mark
@@ -308,22 +380,51 @@ fn parseRulesWithUserAgent(
},
}
},
+ .@"content-signal" => {
+ // A group member like allow/disallow, so it sets has_rules
+ // (a following `User-agent:` opens a new group), but it is
+ // captured into a separate list and never reaches isAllowed.
+ defer state.has_rules = true;
+
+ switch (state.entry) {
+ .in_our_entry => try appendContentSignals(allocator, &content_signals, value),
+ .in_wildcard_entry => try appendContentSignals(allocator, &wildcard_content_signals, value),
+ .in_other_entry, .not_in_entry => {},
+ }
+ },
}
}
// If we have rules for our specific User-Agent, we will use those rules.
// If we don't have any rules, we fallback to using the wildcard ("*") rules.
- if (rules.items.len > 0) {
+ const out_rules = if (rules.items.len > 0) blk: {
freeRulesInList(allocator, wildcard_rules.items);
- return try rules.toOwnedSlice(allocator);
- } else {
+ break :blk try rules.toOwnedSlice(allocator);
+ } else blk: {
freeRulesInList(allocator, rules.items);
- return try wildcard_rules.toOwnedSlice(allocator);
+ break :blk try wildcard_rules.toOwnedSlice(allocator);
+ };
+ errdefer {
+ freeRulesInList(allocator, out_rules);
+ allocator.free(out_rules);
}
+
+ // Content signals follow the same specific-else-wildcard precedence,
+ // chosen independently of the rule selection above.
+ const out_signals = if (content_signals.items.len > 0) blk: {
+ freeContentSignals(allocator, wildcard_content_signals.items);
+ break :blk try content_signals.toOwnedSlice(allocator);
+ } else blk: {
+ freeContentSignals(allocator, content_signals.items);
+ break :blk try wildcard_content_signals.toOwnedSlice(allocator);
+ };
+
+ return .{ .rules = out_rules, .content_signals = out_signals };
}
pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: []const u8) !Robots {
- const rules = try parseRulesWithUserAgent(allocator, user_agent, bytes);
+ const parsed = try parseRulesWithUserAgent(allocator, user_agent, bytes);
+ const rules = parsed.rules;
// sort by order once.
std.mem.sort(Rule, rules, {}, struct {
@@ -357,12 +458,14 @@ pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: []
}
}.lessThan);
- return .{ .rules = rules };
+ return .{ .rules = rules, .content_signals = parsed.content_signals };
}
pub fn deinit(self: *Robots, allocator: std.mem.Allocator) void {
freeRulesInList(allocator, self.rules);
allocator.free(self.rules);
+ freeContentSignals(allocator, self.content_signals);
+ allocator.free(self.content_signals);
}
/// There are rules for how the pattern in robots.txt should be matched.
@@ -476,10 +579,13 @@ test "Robots: simple robots.txt" {
\\
;
- const rules = try parseRulesWithUserAgent(allocator, "GoogleBot", file);
+ const parsed = try parseRulesWithUserAgent(allocator, "GoogleBot", file);
+ const rules = parsed.rules;
defer {
freeRulesInList(allocator, rules);
allocator.free(rules);
+ freeContentSignals(allocator, parsed.content_signals);
+ allocator.free(parsed.content_signals);
}
try std.testing.expectEqual(1, rules.len);
@@ -1003,3 +1109,101 @@ test "Robots: blank lines don't end entries" {
try std.testing.expect(robots.isAllowed("/admin/") == false);
try std.testing.expect(robots.isAllowed("/public/") == true);
}
+
+test "Robots: content-signal captured and gating unaffected" {
+ const allocator = std.testing.allocator;
+
+ var robots = try Robots.fromBytes(allocator, "MyBot",
+ \\User-agent: *
+ \\Content-Signal: ai-train=no, search=yes, ai-input=yes
+ \\Disallow: /private/
+ \\
+ );
+ defer robots.deinit(allocator);
+
+ try std.testing.expectEqual(3, robots.content_signals.len);
+ try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name);
+ try std.testing.expectEqualStrings("no", robots.content_signals[0].value);
+ try std.testing.expectEqualStrings("search", robots.content_signals[1].name);
+ try std.testing.expectEqualStrings("yes", robots.content_signals[1].value);
+ try std.testing.expectEqualStrings("ai-input", robots.content_signals[2].name);
+
+ // The whole point: Content-Signal is advisory metadata, it must not move
+ // the crawl gate. Disallow/Allow still decide isAllowed.
+ try std.testing.expect(robots.isAllowed("/private/page") == false);
+ try std.testing.expect(robots.isAllowed("/public/page") == true);
+}
+
+test "Robots: content-signal mixed casing is normalized" {
+ const allocator = std.testing.allocator;
+
+ var robots = try Robots.fromBytes(allocator, "MyBot",
+ \\User-agent: *
+ \\Content-Signal: AI-Train=NO
+ \\
+ );
+ defer robots.deinit(allocator);
+
+ try std.testing.expectEqual(1, robots.content_signals.len);
+ try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name);
+ try std.testing.expectEqualStrings("no", robots.content_signals[0].value);
+}
+
+test "Robots: content-signal absent yields empty list" {
+ const allocator = std.testing.allocator;
+
+ var robots = try Robots.fromBytes(allocator, "MyBot",
+ \\User-agent: *
+ \\Disallow: /admin/
+ \\
+ );
+ defer robots.deinit(allocator);
+
+ try std.testing.expectEqual(0, robots.content_signals.len);
+ try std.testing.expect(robots.isAllowed("/admin/") == false);
+}
+
+test "Robots: content-signal prefers specific user-agent over wildcard" {
+ const allocator = std.testing.allocator;
+
+ const file =
+ \\User-agent: *
+ \\Content-Signal: ai-train=yes
+ \\
+ \\User-agent: MyBot
+ \\Content-Signal: ai-train=no
+ \\
+ ;
+
+ var mine = try Robots.fromBytes(allocator, "MyBot", file);
+ defer mine.deinit(allocator);
+ try std.testing.expectEqual(1, mine.content_signals.len);
+ try std.testing.expectEqualStrings("no", mine.content_signals[0].value);
+
+ var other = try Robots.fromBytes(allocator, "OtherBot", file);
+ defer other.deinit(allocator);
+ try std.testing.expectEqual(1, other.content_signals.len);
+ try std.testing.expectEqualStrings("yes", other.content_signals[0].value);
+}
+
+test "Robots: RobotStore.getContentSignals round-trips" {
+ const allocator = std.testing.allocator;
+
+ var store = RobotStore.init(allocator);
+ defer store.deinit();
+
+ const robots = try store.robotsFromBytes("MyBot",
+ \\User-agent: *
+ \\Content-Signal: ai-train=no
+ \\
+ );
+ try store.put("https://example.com/robots.txt", robots);
+
+ const signals = store.getContentSignals("https://example.com/robots.txt").?;
+ try std.testing.expectEqual(1, signals.len);
+ try std.testing.expectEqualStrings("ai-train", signals[0].name);
+ try std.testing.expectEqualStrings("no", signals[0].value);
+
+ // Unknown host has no stored robots.
+ try std.testing.expectEqual(null, store.getContentSignals("https://other.com/robots.txt"));
+}