Lightpanda: the headless browser designed for AI and automation
Surface server-side agent-discovery signals (Link header + Content-Signal)
Navid EMADb0a087450dee
diff --git a/src/browser/structured_data.zig b/src/browser/structured_data.zigindex 09cf72a2a..21e063b86 100644--- a/src/browser/structured_data.zig+++ b/src/browser/structured_data.zig@@ -39,6 +39,13 @@ pub const AlternateLink = struct {title: ?[]const u8,};+/// A relation discovered in the HTTP `Link:` response header (RFC 8288),+/// restricted to the registered relations an agent can act on.+pub const LinkRel = struct {+ rel: []const u8,+ href: []const u8,+};+pub const StructuredData = struct {json_ld: []const []const u8,open_graph: []const Property,@@ -46,6 +53,7 @@ pub const StructuredData = struct {meta: []const Property,links: []const Property,alternate: []const AlternateLink,+ link_headers: []const LinkRel,pub fn jsonStringify(self: *const StructuredData, jw: anytype) !void {try jw.beginObject();@@ -89,6 +97,23 @@ pub const StructuredData = struct {try jw.endArray();}+ // Emitted only when present so existing consumers of the CDP/MCP+ // structured-data shape see no new key on sites without a `Link:`+ // response header.+ if (self.link_headers.len > 0) {+ try jw.objectField("linkHeaders");+ try jw.beginArray();+ for (self.link_headers) |lh| {+ try jw.beginObject();+ try jw.objectField("rel");+ try jw.write(lh.rel);+ try jw.objectField("href");+ try jw.write(lh.href);+ try jw.endObject();+ }+ try jw.endArray();+ }+try jw.endObject();}};@@ -144,6 +169,7 @@ pub fn collectStructuredData(var meta: std.ArrayList(Property) = .empty;var links: std.ArrayList(Property) = .empty;var alternate: std.ArrayList(AlternateLink) = .empty;+ var link_headers: std.ArrayList(LinkRel) = .empty;// Extract language from the root <html> element.if (root.is(Element)) |root_el| {@@ -182,6 +208,10 @@ pub fn collectStructuredData(}}+ // The `Link:` response header lives on the frame, not in the DOM, so it is+ // collected separately from the navigation's retained headers.+ try collectLinkHeaders(arena, frame, &link_headers);+return .{.json_ld = json_ld.items,.open_graph = open_graph.items,@@ -189,9 +219,133 @@ pub fn collectStructuredData(.meta = meta.items,.links = links.items,.alternate = alternate.items,+ .link_headers = link_headers.items,};}+/// Registered link relations we surface from the HTTP `Link:` response header+/// (RFC 8288). `service-doc`/`service-desc` come from RFC 8631; `api` is in the+/// IANA Link Relations registry. These let an agent discover a site's developer+/// docs / API description without scraping.+const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" };++/// Parse every `Link:` response header retained on the frame, surfacing only the+/// registered, agent-useful relations. Relative targets are resolved against the+/// frame's base URL, mirroring `collectLink`.+fn collectLinkHeaders(+ arena: Allocator,+ frame: *Frame,+ out: *std.ArrayList(LinkRel),+) !void {+ for (frame._http_headers.items) |header| {+ if (!std.ascii.eqlIgnoreCase(header.name, "link")) continue;++ var it: LinkHeaderIterator = .{ .value = header.value };+ while (it.next()) |entry| {+ const rel_value = entry.rel orelse continue;++ // The `rel` parameter is a space-separated list of relation types.+ var rel_it = std.mem.tokenizeAny(u8, rel_value, " \t");+ while (rel_it.next()) |rel_token| {+ const known = for (header_link_rels) |candidate| {+ if (std.ascii.eqlIgnoreCase(rel_token, candidate)) break candidate;+ } else continue;++ const href = URL.resolve(arena, frame.base(), entry.target, .{ .encoding = frame.charset }) catch entry.target;++ // Drop duplicate (rel, href) pairs — a target can legitimately+ // carry the same relation across repeated headers.+ const dup = for (out.items) |existing| {+ if (std.mem.eql(u8, existing.rel, known) and std.mem.eql(u8, existing.href, href)) break true;+ } else false;+ if (!dup) try out.append(arena, .{ .rel = known, .href = href });+ }+ }+ }+}++/// One `< target >; param=value; ...` entry of an RFC 8288 `Link` header value.+const LinkHeaderEntry = struct {+ target: []const u8,+ rel: ?[]const u8,+};++/// Splits a `Link:` header field value into its comma-separated link-values,+/// skipping commas that fall inside the `<...>` target or a `"..."` quoted+/// parameter value (RFC 8288 §3).+const LinkHeaderIterator = struct {+ value: []const u8,+ i: usize = 0,++ fn next(self: *LinkHeaderIterator) ?LinkHeaderEntry {+ const v = self.value;++ // Loop (rather than recurse) over malformed segments so stack usage is+ // independent of the attacker-controlled header length.+ while (true) {+ // Skip separators and whitespace between link-values. The comma that+ // ended the previous link-value is consumed here.+ while (self.i < v.len and (std.ascii.isWhitespace(v[self.i]) or v[self.i] == ',')) : (self.i += 1) {}+ if (self.i >= v.len) return null;++ // A link-value must begin with the angle-bracketed target.+ if (v[self.i] != '<') {+ self.skipToNextComma();+ continue;+ }+ const target_start = self.i + 1;+ const gt = std.mem.indexOfScalarPos(u8, v, target_start, '>') orelse {+ self.i = v.len;+ return null;+ };+ const target = v[target_start..gt];+ self.i = gt + 1;++ // Parameters run to the next top-level comma, left at `self.i`.+ const params_start = self.i;+ self.skipToNextComma();+ return .{ .target = target, .rel = extractRel(v[params_start..self.i]) };+ }+ }++ // Advance until `self.i` sits on the next top-level comma (one outside a+ // `"..."` quoted-string, honouring RFC 7230 backslash escapes) or the end+ // of the value. The comma itself is left for the leading-separator skip.+ fn skipToNextComma(self: *LinkHeaderIterator) void {+ const v = self.value;+ var in_quotes = false;+ while (self.i < v.len) : (self.i += 1) {+ const c = v[self.i];+ if (c == '\\' and in_quotes) {+ // Skip the escaped byte; guard a trailing backslash.+ if (self.i + 1 < v.len) self.i += 1;+ } else if (c == '"') {+ in_quotes = !in_quotes;+ } else if (c == ',' and !in_quotes) {+ return;+ }+ }+ }+};++/// Return the (unquoted) value of the `rel` parameter from a link-value's+/// parameter list, or null when absent.+fn extractRel(params: []const u8) ?[]const u8 {+ var it = std.mem.splitScalar(u8, params, ';');+ while (it.next()) |raw_param| {+ const param = std.mem.trim(u8, raw_param, &std.ascii.whitespace);+ const eq = std.mem.indexOfScalar(u8, param, '=') orelse continue;+ if (!std.ascii.eqlIgnoreCase(std.mem.trim(u8, param[0..eq], &std.ascii.whitespace), "rel")) continue;++ var val = std.mem.trim(u8, param[eq + 1 ..], &std.ascii.whitespace);+ if (val.len >= 2 and val[0] == '"' and val[val.len - 1] == '"') {+ val = val[1 .. val.len - 1];+ }+ return val;+ }+ return null;+}+fn collectJsonLd(el: *Element,arena: Allocator,@@ -484,6 +638,78 @@ test "structured_data: charset and http-equiv" {try testing.expectEqual("text/html; charset=utf-8", findProperty(data.meta, "Content-Type").?);}+test "structured_data: parseLinkHeader - single link with quoted rel" {+ var it: LinkHeaderIterator = .{ .value =+ \\<https://api.example.com/openapi.json>; rel="service-desc"+ };+ const entry = it.next().?;+ try testing.expectString("https://api.example.com/openapi.json", entry.target);+ try testing.expectString("service-desc", entry.rel.?);+ try testing.expectEqual(null, it.next());+}++test "structured_data: parseLinkHeader - multiple links and unquoted rel" {+ var it: LinkHeaderIterator = .{ .value =+ \\<https://docs.example.com/>; rel=service-doc, </style.css>; rel="stylesheet"; type="text/css"+ };+ const first = it.next().?;+ try testing.expectString("https://docs.example.com/", first.target);+ try testing.expectString("service-doc", first.rel.?);++ const second = it.next().?;+ try testing.expectString("/style.css", second.target);+ // A comma inside the quoted type param must not split the link-value.+ try testing.expectString("stylesheet", second.rel.?);++ try testing.expectEqual(null, it.next());+}++test "structured_data: parseLinkHeader - escaped quote in param does not split link" {+ // A backslash-escaped quote inside an earlier param must not terminate the+ // quoted-string, so the comma inside it is not treated as a separator.+ var it: LinkHeaderIterator = .{ .value =+ \\<https://a/>; title="x\",y"; rel="service-doc"+ };+ const entry = it.next().?;+ try testing.expectString("https://a/", entry.target);+ try testing.expectString("service-doc", entry.rel.?);+ try testing.expectEqual(null, it.next());+}++test "structured_data: link headers from response" {+ const frame = try testing.test_session.createPage();+ defer testing.test_session.removePage();++ // Stand in for what frameHeaderDoneCallback records from the navigation.+ try frame._http_headers.append(frame.arena, .{ .name = "Link", .value =+ \\<https://docs.example.com/>; rel="service-doc"+ });+ try frame._http_headers.append(frame.arena, .{ .name = "link", .value =+ \\<https://api.example.com/spec>; rel="service-desc", </css/site.css>; rel="stylesheet"+ });++ const doc = frame.window._document;+ const div = try doc.createElement("div", null, frame);+ try frame.parseHtmlAsChildren(div.asNode(), "<title>Example</title>");++ const data = try collectStructuredData(div.asNode(), frame.call_arena, frame);++ // service-doc + service-desc are surfaced; stylesheet is filtered out.+ try testing.expectEqual(2, data.link_headers.len);+ try testing.expectString("service-doc", data.link_headers[0].rel);+ try testing.expectString("https://docs.example.com/", data.link_headers[0].href);+ try testing.expectString("service-desc", data.link_headers[1].rel);+ try testing.expectString("https://api.example.com/spec", data.link_headers[1].href);+}++test "structured_data: no link header yields empty list" {+ const data = try testStructuredData(+ \\<link rel="canonical" href="https://example.com">+ );+ defer testing.test_session.removePage();+ try testing.expectEqual(0, data.link_headers.len);+}+test "structured_data: mixed content" {const data = try testStructuredData(\\<title>My Site</title>diff --git a/src/cdp/domains/lp.zig b/src/cdp/domains/lp.zigindex b3b3aec0a..dd8fdd991 100644--- a/src/cdp/domains/lp.zig+++ b/src/cdp/domains/lp.zig@@ -23,6 +23,7 @@ const CDP = @import("../CDP.zig");const Node = @import("../Node.zig");const DOMNode = @import("../../browser/webapi/Node.zig");+const Robots = @import("../../network/Robots.zig");const markdown = lp.markdown;const SemanticTree = lp.SemanticTree;@@ -36,6 +37,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {getInteractiveElements,getNodeDetails,getStructuredData,+ getContentSignal,detectForms,clickNode,fillNode,@@ -51,6 +53,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {.getInteractiveElements => return getInteractiveElements(cmd),.getNodeDetails => return getNodeDetails(cmd),.getStructuredData => return getStructuredData(cmd),+ .getContentSignal => return getContentSignal(cmd),.detectForms => return detectForms(cmd),.clickNode => return clickNode(cmd),.fillNode => return fillNode(cmd),@@ -201,6 +204,27 @@ fn getStructuredData(cmd: anytype) !void {}, .{});}+// Advisory `Content-Signal` robots.txt preferences for the current document's+// host (https://contentsignals.org). `available` is false when no robots.txt+// has been fetched for the host — note this is the case unless `obey_robots`+// is enabled, since the robots layer is what populates the store.+fn getContentSignal(cmd: anytype) !void {+ const bc = cmd.browser_context orelse return error.NoBrowserContext;+ const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded;+ const network = bc.cdp.browser.http_client.network;++ const empty: []const Robots.ContentSignal = &.{};+ const robots_url = lp.URL.getRobotsUrl(cmd.arena, frame.url) catch {+ return cmd.sendResult(.{ .available = false, .contentSignals = empty }, .{});+ };++ const signals = network.robot_store.getContentSignals(robots_url);+ return cmd.sendResult(.{+ .available = signals != null,+ .contentSignals = signals orelse empty,+ }, .{});+}+fn detectForms(cmd: anytype) !void {const bc = cmd.browser_context orelse return error.NoBrowserContext;const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded;@@ -410,6 +434,25 @@ test "cdp.lp: getStructuredData" {try testing.expect(result.get("structuredData") != null);}+test "cdp.lp: getContentSignal" {+ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{});+ _ = try bc.session.createPage();++ try ctx.processMessage(.{+ .id = 1,+ .method = "LP.getContentSignal",+ });++ // No robots.txt fetched for the page, so the directive is reported absent+ // rather than erroring — the command is still wired and returns the shape.+ const result = (try ctx.getSentMessage(0)).?.object.get("result").?.object;+ try testing.expectEqual(false, result.get("available").?.bool);+ try testing.expect(result.get("contentSignals") != null);+}+test "cdp.lp: action tools" {var ctx = try testing.context();defer ctx.deinit();diff --git a/src/network/Robots.zig b/src/network/Robots.zigindex 202e0a5c7..423a308cf 100644--- a/src/network/Robots.zig+++ b/src/network/Robots.zig@@ -72,11 +72,20 @@ pub const Key = enum {@"user-agent",allow,disallow,+ @"content-signal",+};++/// A declared content-usage preference from a `Content-Signal:` robots.txt+/// directive (https://contentsignals.org), e.g. name="ai-train" value="no".+/// Advisory metadata surfaced to the agent — it never affects crawl gating.+pub const ContentSignal = struct {+ name: []const u8,+ value: []const u8,};/// https://www.rfc-editor.org/rfc/rfc9309.htmlpub const Robots = @This();-pub const empty: Robots = .{ .rules = &.{} };+pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };pub const RobotStore = struct {const RobotsEntry = union(enum) {@@ -156,6 +165,21 @@ pub const RobotStore = struct {try self.map.put(self.allocator, duped, .{ .present = robots });}+ /// Advisory `Content-Signal` preferences declared for `url`'s host, or null+ /// when no robots.txt has been fetched/stored for it. The returned slice is+ /// owned by the store; callers must read it synchronously (and dupe to keep+ /// it) and must never free it.+ pub fn getContentSignals(self: *RobotStore, url: []const u8) ?[]const ContentSignal {+ self.mutex.lock();+ defer self.mutex.unlock();++ const entry = self.map.get(url) orelse return null;+ return switch (entry) {+ .present => |robots| robots.content_signals,+ .absent => null,+ };+ }+pub fn putAbsent(self: *RobotStore, url: []const u8) !void {self.mutex.lock();defer self.mutex.unlock();@@ -166,6 +190,15 @@ pub const RobotStore = struct {};rules: []const Rule,+content_signals: []const ContentSignal = &.{},++/// Result of parsing a robots.txt for one user-agent: the applicable crawl+/// rules plus any advisory `Content-Signal` preferences. The two are kept+/// distinct so content signals can never reach `isAllowed`.+const ParseResult = struct {+ rules: []Rule,+ content_signals: []ContentSignal,+};const State = struct {entry: enum {@@ -186,17 +219,56 @@ fn freeRulesInList(allocator: std.mem.Allocator, rules: []const Rule) void {}}+fn freeContentSignals(allocator: std.mem.Allocator, signals: []const ContentSignal) void {+ for (signals) |signal| {+ allocator.free(signal.name);+ allocator.free(signal.value);+ }+}++/// Parse a `Content-Signal:` value — a comma-separated list of `name=value`+/// pairs (e.g. `ai-train=no, search=yes`) — into the given list. Names and+/// values are lower-cased and owned by `allocator`.+fn appendContentSignals(+ allocator: std.mem.Allocator,+ list: *std.ArrayList(ContentSignal),+ value: []const u8,+) !void {+ var iter = std.mem.splitScalar(u8, value, ',');+ while (iter.next()) |raw_pair| {+ const pair = std.mem.trim(u8, raw_pair, &std.ascii.whitespace);+ const eq = std.mem.indexOfScalar(u8, pair, '=') orelse continue;++ const name_raw = std.mem.trim(u8, pair[0..eq], &std.ascii.whitespace);+ const value_raw = std.mem.trim(u8, pair[eq + 1 ..], &std.ascii.whitespace);+ if (name_raw.len == 0) continue;++ const name = try std.ascii.allocLowerString(allocator, name_raw);+ errdefer allocator.free(name);+ const val = try std.ascii.allocLowerString(allocator, value_raw);+ errdefer allocator.free(val);++ try list.append(allocator, .{ .name = name, .value = val });+ }+}+fn parseRulesWithUserAgent(allocator: std.mem.Allocator,user_agent: []const u8,raw_bytes: []const u8,-) ![]Rule {+) !ParseResult {var rules: std.ArrayList(Rule) = .empty;defer rules.deinit(allocator);var wildcard_rules: std.ArrayList(Rule) = .empty;defer wildcard_rules.deinit(allocator);+ var content_signals: std.ArrayList(ContentSignal) = .empty;+ defer content_signals.deinit(allocator);++ var wildcard_content_signals: std.ArrayList(ContentSignal) = .empty;+ defer wildcard_content_signals.deinit(allocator);+var state: State = .{ .entry = .not_in_entry, .has_rules = false };// https://en.wikipedia.org/wiki/Byte_order_mark@@ -308,22 +380,51 @@ fn parseRulesWithUserAgent(},}},+ .@"content-signal" => {+ // A group member like allow/disallow, so it sets has_rules+ // (a following `User-agent:` opens a new group), but it is+ // captured into a separate list and never reaches isAllowed.+ defer state.has_rules = true;++ switch (state.entry) {+ .in_our_entry => try appendContentSignals(allocator, &content_signals, value),+ .in_wildcard_entry => try appendContentSignals(allocator, &wildcard_content_signals, value),+ .in_other_entry, .not_in_entry => {},+ }+ },}}// If we have rules for our specific User-Agent, we will use those rules.// If we don't have any rules, we fallback to using the wildcard ("*") rules.- if (rules.items.len > 0) {+ const out_rules = if (rules.items.len > 0) blk: {freeRulesInList(allocator, wildcard_rules.items);- return try rules.toOwnedSlice(allocator);- } else {+ break :blk try rules.toOwnedSlice(allocator);+ } else blk: {freeRulesInList(allocator, rules.items);- return try wildcard_rules.toOwnedSlice(allocator);+ break :blk try wildcard_rules.toOwnedSlice(allocator);+ };+ errdefer {+ freeRulesInList(allocator, out_rules);+ allocator.free(out_rules);}++ // Content signals follow the same specific-else-wildcard precedence,+ // chosen independently of the rule selection above.+ const out_signals = if (content_signals.items.len > 0) blk: {+ freeContentSignals(allocator, wildcard_content_signals.items);+ break :blk try content_signals.toOwnedSlice(allocator);+ } else blk: {+ freeContentSignals(allocator, content_signals.items);+ break :blk try wildcard_content_signals.toOwnedSlice(allocator);+ };++ return .{ .rules = out_rules, .content_signals = out_signals };}pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: []const u8) !Robots {- const rules = try parseRulesWithUserAgent(allocator, user_agent, bytes);+ const parsed = try parseRulesWithUserAgent(allocator, user_agent, bytes);+ const rules = parsed.rules;// sort by order once.std.mem.sort(Rule, rules, {}, struct {@@ -357,12 +458,14 @@ pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: []}}.lessThan);- return .{ .rules = rules };+ return .{ .rules = rules, .content_signals = parsed.content_signals };}pub fn deinit(self: *Robots, allocator: std.mem.Allocator) void {freeRulesInList(allocator, self.rules);allocator.free(self.rules);+ freeContentSignals(allocator, self.content_signals);+ allocator.free(self.content_signals);}/// There are rules for how the pattern in robots.txt should be matched.@@ -476,10 +579,13 @@ test "Robots: simple robots.txt" {\\;- const rules = try parseRulesWithUserAgent(allocator, "GoogleBot", file);+ const parsed = try parseRulesWithUserAgent(allocator, "GoogleBot", file);+ const rules = parsed.rules;defer {freeRulesInList(allocator, rules);allocator.free(rules);+ freeContentSignals(allocator, parsed.content_signals);+ allocator.free(parsed.content_signals);}try std.testing.expectEqual(1, rules.len);@@ -1003,3 +1109,101 @@ test "Robots: blank lines don't end entries" {try std.testing.expect(robots.isAllowed("/admin/") == false);try std.testing.expect(robots.isAllowed("/public/") == true);}++test "Robots: content-signal captured and gating unaffected" {+ const allocator = std.testing.allocator;++ var robots = try Robots.fromBytes(allocator, "MyBot",+ \\User-agent: *+ \\Content-Signal: ai-train=no, search=yes, ai-input=yes+ \\Disallow: /private/+ \\+ );+ defer robots.deinit(allocator);++ try std.testing.expectEqual(3, robots.content_signals.len);+ try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name);+ try std.testing.expectEqualStrings("no", robots.content_signals[0].value);+ try std.testing.expectEqualStrings("search", robots.content_signals[1].name);+ try std.testing.expectEqualStrings("yes", robots.content_signals[1].value);+ try std.testing.expectEqualStrings("ai-input", robots.content_signals[2].name);++ // The whole point: Content-Signal is advisory metadata, it must not move+ // the crawl gate. Disallow/Allow still decide isAllowed.+ try std.testing.expect(robots.isAllowed("/private/page") == false);+ try std.testing.expect(robots.isAllowed("/public/page") == true);+}++test "Robots: content-signal mixed casing is normalized" {+ const allocator = std.testing.allocator;++ var robots = try Robots.fromBytes(allocator, "MyBot",+ \\User-agent: *+ \\Content-Signal: AI-Train=NO+ \\+ );+ defer robots.deinit(allocator);++ try std.testing.expectEqual(1, robots.content_signals.len);+ try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name);+ try std.testing.expectEqualStrings("no", robots.content_signals[0].value);+}++test "Robots: content-signal absent yields empty list" {+ const allocator = std.testing.allocator;++ var robots = try Robots.fromBytes(allocator, "MyBot",+ \\User-agent: *+ \\Disallow: /admin/+ \\+ );+ defer robots.deinit(allocator);++ try std.testing.expectEqual(0, robots.content_signals.len);+ try std.testing.expect(robots.isAllowed("/admin/") == false);+}++test "Robots: content-signal prefers specific user-agent over wildcard" {+ const allocator = std.testing.allocator;++ const file =+ \\User-agent: *+ \\Content-Signal: ai-train=yes+ \\+ \\User-agent: MyBot+ \\Content-Signal: ai-train=no+ \\+ ;++ var mine = try Robots.fromBytes(allocator, "MyBot", file);+ defer mine.deinit(allocator);+ try std.testing.expectEqual(1, mine.content_signals.len);+ try std.testing.expectEqualStrings("no", mine.content_signals[0].value);++ var other = try Robots.fromBytes(allocator, "OtherBot", file);+ defer other.deinit(allocator);+ try std.testing.expectEqual(1, other.content_signals.len);+ try std.testing.expectEqualStrings("yes", other.content_signals[0].value);+}++test "Robots: RobotStore.getContentSignals round-trips" {+ const allocator = std.testing.allocator;++ var store = RobotStore.init(allocator);+ defer store.deinit();++ const robots = try store.robotsFromBytes("MyBot",+ \\User-agent: *+ \\Content-Signal: ai-train=no+ \\+ );+ try store.put("https://example.com/robots.txt", robots);++ const signals = store.getContentSignals("https://example.com/robots.txt").?;+ try std.testing.expectEqual(1, signals.len);+ try std.testing.expectEqualStrings("ai-train", signals[0].name);+ try std.testing.expectEqualStrings("no", signals[0].value);++ // Unknown host has no stored robots.+ try std.testing.expectEqual(null, store.getContentSignals("https://other.com/robots.txt"));+}