Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(crawl): stop the crawler treating every IPv4 host as private and looping on redirecting pages

nirholas9ecfda0bace1
diff --git a/tests/agent-crawler-guard.test.js b/tests/agent-crawler-guard.test.js
new file mode 100644
index 000000000..673ee9eac
--- /dev/null
+++ b/tests/agent-crawler-guard.test.js
@@ -0,0 +1,29 @@
+// The Crawl worker's network fence (workers/agent-crawler/guard.js): every
+// address a crawled page reaches is checked here before the browser connects.
+import { describe, expect, it } from 'vitest';
+import { guardHost, isBlockedAddress } from '../workers/agent-crawler/guard.js';
+
+describe('isBlockedAddress', () => {
+ it.each([
+ '10.0.0.1', '127.0.0.1', '169.254.169.254', '172.16.5.4', '192.168.0.10', '100.64.1.1', '0.0.0.0', '224.0.0.251',
+ '198.18.0.1', '::1', '::', 'fd12::1', 'fe80::1', '::ffff:127.0.0.1', '::ffff:10.0.0.1', 'not-an-ip',
+ ])('refuses %s', (ip) => expect(isBlockedAddress(ip)).toBe(true));
+
+ // A rule for the IPv4-mapped range once matched every plain IPv4 address and
+ // refused the entire public web; these pin that regression.
+ it.each(['198.35.26.224', '8.8.8.8', '1.1.1.1', '172.32.0.1', '2620:0:863:ed1a::1', '::ffff:8.8.8.8'])('allows %s', (ip) => {
+ expect(isBlockedAddress(ip)).toBe(false);
+ });
+});
+
+describe('guardHost', () => {
+ it('refuses internal names and private literals without a lookup', async () => {
+ for (const h of ['localhost', 'metadata', 'metadata.google.internal', 'nas.local', 'db.internal', '10.1.1.1', '[::1]', '']) {
+ expect(await guardHost(h), h).toBe(false);
+ }
+ });
+
+ it('allows a public literal', async () => {
+ expect(await guardHost('8.8.8.8')).toBe(true);
+ });
+});
diff --git a/workers/agent-crawler/Dockerfile b/workers/agent-crawler/Dockerfile
index a1ae638f5..470955a3b 100644
--- a/workers/agent-crawler/Dockerfile
+++ b/workers/agent-crawler/Dockerfile
@@ -13,7 +13,7 @@ WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci --omit=dev
-COPY index.js crawler.js frontier.js extract.js net.js seeds.js ./
+COPY index.js crawler.js frontier.js extract.js guard.js robots.js seeds.js ./
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
ENV NODE_ENV=production \
diff --git a/workers/agent-crawler/crawler.js b/workers/agent-crawler/crawler.js
index 9e8990ac9..211a0aa2e 100644
--- a/workers/agent-crawler/crawler.js
+++ b/workers/agent-crawler/crawler.js
@@ -10,7 +10,8 @@ import { readPage, revealLink, visibleLinks, CONSENT_CSS } from './extract.js';
import {
Frontier, canonical, gistOf, hostOf, relevance, scoreLink, thoughtFor, topicTerms,
} from './frontier.js';
-import { USER_AGENT, claimDomain, domainReadyIn, guardHost, robotsCheck } from './net.js';
+import { USER_AGENT, guardHost } from './guard.js';
+import { claimDomain, domainReadyIn, robotsCheck } from './robots.js';
import { searchSeeds } from './seeds.js';
export const VIEWPORT = { width: 1200, height: 750 };
@@ -36,6 +37,7 @@ export class Crawler {
this.log = (msg) => log(`[${this.name}] ${msg}`);
this.frontier = new Frontier();
this.visited = new Set();
+ this.bounces = new Map(); // page -> links of it that redirected back to it
this.domainCounts = new Map();
this.seq = 0;
this.stopped = false;
@@ -165,8 +167,15 @@ export class Crawler {
}
}
+ // Start pages are re-read on every shift even if the corpus already holds
+ // them: they are where the owner pointed the agent, and their links are the
+ // freshest leads it has. The API files a re-read page only once.
async seed() {
- for (const url of this.seeds) this.frontier.add(url, 50, null, 'start page');
+ for (const url of this.seeds) {
+ const key = canonical(url);
+ if (key) this.visited.delete(key);
+ this.frontier.add(url, 50, null, 'start page');
+ }
if (!this.frontier.size) await this.searchForLeads();
}
@@ -178,10 +187,15 @@ export class Crawler {
return urls.length;
}
+ // Called when pickNext() found nothing. Every path either adds an unvisited
+ // lead or really waits, so run() can never spin without yielding.
async reseedOrRest() {
- const sinceReseed = Date.now() - this.lastReseedAt;
- if (sinceReseed > 120_000) {
- for (const url of this.seeds) if (!this.visited.has(canonical(url))) this.frontier.add(url, 50);
+ for (const item of [...this.frontier.items.values()]) {
+ if (this.visited.has(item.url)) this.frontier.drop(item.url);
+ }
+ if (Date.now() - this.lastReseedAt > 120_000) {
+ this.lastReseedAt = Date.now();
+ for (const url of this.seeds) if (!this.visited.has(canonical(url))) this.frontier.add(url, 50, null, 'start page');
if (this.frontier.size || (await this.searchForLeads())) return;
}
this.status = 'resting';
@@ -211,6 +225,20 @@ export class Crawler {
return item;
}
+ // Two links from one page that both bounced back to it mean the rest of
+ // its links will too (a hash-routed docs shell, a forced locale redirect):
+ // drop them all so the agent stops walking in circles on one screen.
+ noteBounce(home) {
+ const n = (this.bounces.get(home) || 0) + 1;
+ this.bounces.set(home, n);
+ if (n !== 2) return;
+ let dropped = 0;
+ for (const item of [...this.frontier.items.values()]) {
+ if (item.from === home) { this.frontier.drop(item.url); dropped += 1; }
+ }
+ if (dropped) this.log(`${home} redirects its links back to itself; dropped ${dropped} leads`);
+ }
+
async pushStep(step, extra = {}) {
this.seq += 1;
return this.api.push({ agentId: this.agentId, step: { ...step, seq: this.seq }, ...extra });
@@ -278,6 +306,13 @@ export class Crawler {
await this.page.waitForLoadState('load', { timeout: 6000 }).catch(() => {});
const finalUrl = canonical(this.page.url()) || url;
+ // Redirected onto a page this agent already read (a docs shell that
+ // bounces every deep link back to itself, a paywall home page): nothing
+ // new to read here, so let the frontier choose without dwelling.
+ if (finalUrl !== url && this.visited.has(finalUrl)) {
+ this.noteBounce(finalUrl);
+ return null;
+ }
this.visited.add(finalUrl);
const finalHost = hostOf(finalUrl);
this.domainCounts.set(finalHost, (this.domainCounts.get(finalHost) || 0) + 1);
diff --git a/workers/agent-crawler/net.js b/workers/agent-crawler/guard.js
similarity index 53%
rename from workers/agent-crawler/net.js
rename to workers/agent-crawler/guard.js
index b1974e0d5..3b9595103 100644
--- a/workers/agent-crawler/net.js
+++ b/workers/agent-crawler/guard.js
@@ -1,16 +1,13 @@
-// Network fences for a browser that goes wherever links lead.
+// The network fence for a browser that goes wherever links lead.
//
// guardHost(): every request the crawler's pages make is resolved first and
// refused when ANY address it resolves to is private, loopback, link-local or
// reserved. That is what keeps a hostile page (or a mission seed pointing at an
// internal name) from turning the crawler into a probe of the network it runs in.
-//
-// Robots: robots.txt is fetched once per origin and honored, including
-// Crawl-delay, under the crawler's own product token.
+// Dependency-free so tests/agent-crawler-guard.test.js can pin it.
import dns from 'node:dns/promises';
import net from 'node:net';
-import robotsParser from 'robots-parser';
export const BOT_TOKEN = 'three.ws-crawler';
export const USER_AGENT =
@@ -23,9 +20,11 @@ for (const [addr, prefix] of [
['203.0.113.0', 24], ['224.0.0.0', 4], ['240.0.0.0', 4],
]) blocked.addSubnet(addr, prefix, 'ipv4');
for (const [addr, prefix] of [
- ['::', 128], ['::1', 128], ['::ffff:0:0', 96], ['64:ff9b::', 96], ['100::', 64], ['2001:db8::', 32],
- ['fc00::', 7], ['fe80::', 10], ['ff00::', 8],
+ ['::', 128], ['::1', 128], ['100::', 64], ['2001:db8::', 32], ['fc00::', 7], ['fe80::', 10], ['ff00::', 8],
]) blocked.addSubnet(addr, prefix, 'ipv6');
+// No ::ffff:0:0/96 rule on purpose: BlockList already checks an IPv4-mapped
+// address against the IPv4 rules above, and a mapped-range rule would match
+// EVERY plain IPv4 address, refusing the whole public web.
export function isBlockedAddress(ip) {
const family = net.isIP(ip);
@@ -66,51 +65,3 @@ export async function guardedFetch(url, { timeoutMs = 8000, headers = {} } = {})
signal: AbortSignal.timeout(timeoutMs),
});
}
-
-const robotsCache = new Map(); // origin -> { robots, at }
-const ROBOTS_TTL_MS = 60 * 60_000;
-
-async function robotsFor(origin) {
- const hit = robotsCache.get(origin);
- if (hit && Date.now() - hit.at < ROBOTS_TTL_MS) return hit.robots;
- const robotsUrl = `${origin}/robots.txt`;
- let body = '';
- try {
- const res = await guardedFetch(robotsUrl, { timeoutMs: 6000, headers: { accept: 'text/plain' } });
- // RFC 9309: 4xx means no rules; 5xx or a network failure means "assume
- // disallowed for now" so a struggling server is not crawled harder.
- if (res.status >= 500) body = 'User-agent: *\nDisallow: /';
- else if (res.ok) body = (await res.text()).slice(0, 500_000);
- } catch {
- body = 'User-agent: *\nDisallow: /';
- }
- const robots = robotsParser(robotsUrl, body);
- robotsCache.set(origin, { robots, at: Date.now() });
- if (robotsCache.size > 3000) robotsCache.delete(robotsCache.keys().next().value);
- return robots;
-}
-
-// { allowed, delayMs } for one URL under the crawler's token.
-export async function robotsCheck(url) {
- const u = new URL(url);
- const robots = await robotsFor(u.origin);
- const allowed = robots.isAllowed(u.href, BOT_TOKEN) !== false;
- const delay = Number(robots.getCrawlDelay(BOT_TOKEN));
- return { allowed, delayMs: Number.isFinite(delay) && delay > 0 ? Math.min(delay, 30) * 1000 : 0 };
-}
-
-// Per-domain politeness shared by every crawler in the process: at most one
-// request to a domain every gapMs (or its robots Crawl-delay, if longer).
-const domainNext = new Map();
-
-export function domainReadyIn(host) {
- return Math.max(0, (domainNext.get(host) || 0) - Date.now());
-}
-
-export function claimDomain(host, gapMs) {
- domainNext.set(host, Date.now() + gapMs);
- if (domainNext.size > 10_000) {
- const now = Date.now();
- for (const [h, t] of domainNext) if (t < now) domainNext.delete(h);
- }
-}
diff --git a/workers/agent-crawler/robots.js b/workers/agent-crawler/robots.js
new file mode 100644
index 000000000..c807e6ecc
--- /dev/null
+++ b/workers/agent-crawler/robots.js
@@ -0,0 +1,53 @@
+// robots.txt and per-domain politeness. robots.txt is fetched once per origin
+// and honored, including Crawl-delay, under the crawler's own product token.
+
+import robotsParser from 'robots-parser';
+import { BOT_TOKEN, guardedFetch } from './guard.js';
+
+const robotsCache = new Map(); // origin -> { robots, at }
+const ROBOTS_TTL_MS = 60 * 60_000;
+
+async function robotsFor(origin) {
+ const hit = robotsCache.get(origin);
+ if (hit && Date.now() - hit.at < ROBOTS_TTL_MS) return hit.robots;
+ const robotsUrl = `${origin}/robots.txt`;
+ let body = '';
+ try {
+ const res = await guardedFetch(robotsUrl, { timeoutMs: 6000, headers: { accept: 'text/plain' } });
+ // RFC 9309: 4xx means no rules; 5xx or a network failure means "assume
+ // disallowed for now" so a struggling server is not crawled harder.
+ if (res.status >= 500) body = 'User-agent: *\nDisallow: /';
+ else if (res.ok) body = (await res.text()).slice(0, 500_000);
+ } catch {
+ body = 'User-agent: *\nDisallow: /';
+ }
+ const robots = robotsParser(robotsUrl, body);
+ robotsCache.set(origin, { robots, at: Date.now() });
+ if (robotsCache.size > 3000) robotsCache.delete(robotsCache.keys().next().value);
+ return robots;
+}
+
+// { allowed, delayMs } for one URL under the crawler's token.
+export async function robotsCheck(url) {
+ const u = new URL(url);
+ const robots = await robotsFor(u.origin);
+ const allowed = robots.isAllowed(u.href, BOT_TOKEN) !== false;
+ const delay = Number(robots.getCrawlDelay(BOT_TOKEN));
+ return { allowed, delayMs: Number.isFinite(delay) && delay > 0 ? Math.min(delay, 30) * 1000 : 0 };
+}
+
+// Per-domain politeness shared by every crawler in the process: at most one
+// request to a domain every gapMs (or its robots Crawl-delay, if longer).
+const domainNext = new Map();
+
+export function domainReadyIn(host) {
+ return Math.max(0, (domainNext.get(host) || 0) - Date.now());
+}
+
+export function claimDomain(host, gapMs) {
+ domainNext.set(host, Date.now() + gapMs);
+ if (domainNext.size > 10_000) {
+ const now = Date.now();
+ for (const [h, t] of domainNext) if (t < now) domainNext.delete(h);
+ }
+}
diff --git a/workers/agent-crawler/seeds.js b/workers/agent-crawler/seeds.js
index 341a2f181..3355c6470 100644
--- a/workers/agent-crawler/seeds.js
+++ b/workers/agent-crawler/seeds.js
@@ -2,7 +2,7 @@
// walked itself into a dead end: real public search, tried in order until one
// answers. Each source is a free, keyless public endpoint.
-import { guardedFetch } from './net.js';
+import { guardedFetch } from './guard.js';
// DuckDuckGo's HTML endpoint. Result anchors carry the target in the uddg
// parameter of a /l/ redirect. It sometimes answers bots with a 202 challenge