Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(crawl): stop the crawler treating every IPv4 host as private and looping on redirecting pages
nirholas9ecfda0bace1
diff --git a/tests/agent-crawler-guard.test.js b/tests/agent-crawler-guard.test.jsnew file mode 100644index 000000000..673ee9eac--- /dev/null+++ b/tests/agent-crawler-guard.test.js@@ -0,0 +1,29 @@+// The Crawl worker's network fence (workers/agent-crawler/guard.js): every+// address a crawled page reaches is checked here before the browser connects.+import { describe, expect, it } from 'vitest';+import { guardHost, isBlockedAddress } from '../workers/agent-crawler/guard.js';++describe('isBlockedAddress', () => {+ it.each([+ '10.0.0.1', '127.0.0.1', '169.254.169.254', '172.16.5.4', '192.168.0.10', '100.64.1.1', '0.0.0.0', '224.0.0.251',+ '198.18.0.1', '::1', '::', 'fd12::1', 'fe80::1', '::ffff:127.0.0.1', '::ffff:10.0.0.1', 'not-an-ip',+ ])('refuses %s', (ip) => expect(isBlockedAddress(ip)).toBe(true));++ // A rule for the IPv4-mapped range once matched every plain IPv4 address and+ // refused the entire public web; these pin that regression.+ it.each(['198.35.26.224', '8.8.8.8', '1.1.1.1', '172.32.0.1', '2620:0:863:ed1a::1', '::ffff:8.8.8.8'])('allows %s', (ip) => {+ expect(isBlockedAddress(ip)).toBe(false);+ });+});++describe('guardHost', () => {+ it('refuses internal names and private literals without a lookup', async () => {+ for (const h of ['localhost', 'metadata', 'metadata.google.internal', 'nas.local', 'db.internal', '10.1.1.1', '[::1]', '']) {+ expect(await guardHost(h), h).toBe(false);+ }+ });++ it('allows a public literal', async () => {+ expect(await guardHost('8.8.8.8')).toBe(true);+ });+});diff --git a/workers/agent-crawler/Dockerfile b/workers/agent-crawler/Dockerfileindex a1ae638f5..470955a3b 100644--- a/workers/agent-crawler/Dockerfile+++ b/workers/agent-crawler/Dockerfile@@ -13,7 +13,7 @@ WORKDIR /appCOPY package.json package-lock.json ./RUN npm ci --omit=dev-COPY index.js crawler.js frontier.js extract.js net.js seeds.js ./+COPY index.js crawler.js frontier.js extract.js guard.js robots.js seeds.js ./ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwrightENV NODE_ENV=production \diff --git a/workers/agent-crawler/crawler.js b/workers/agent-crawler/crawler.jsindex 9e8990ac9..211a0aa2e 100644--- a/workers/agent-crawler/crawler.js+++ b/workers/agent-crawler/crawler.js@@ -10,7 +10,8 @@ import { readPage, revealLink, visibleLinks, CONSENT_CSS } from './extract.js';import {Frontier, canonical, gistOf, hostOf, relevance, scoreLink, thoughtFor, topicTerms,} from './frontier.js';-import { USER_AGENT, claimDomain, domainReadyIn, guardHost, robotsCheck } from './net.js';+import { USER_AGENT, guardHost } from './guard.js';+import { claimDomain, domainReadyIn, robotsCheck } from './robots.js';import { searchSeeds } from './seeds.js';export const VIEWPORT = { width: 1200, height: 750 };@@ -36,6 +37,7 @@ export class Crawler {this.log = (msg) => log(`[${this.name}] ${msg}`);this.frontier = new Frontier();this.visited = new Set();+ this.bounces = new Map(); // page -> links of it that redirected back to itthis.domainCounts = new Map();this.seq = 0;this.stopped = false;@@ -165,8 +167,15 @@ export class Crawler {}}+ // Start pages are re-read on every shift even if the corpus already holds+ // them: they are where the owner pointed the agent, and their links are the+ // freshest leads it has. The API files a re-read page only once.async seed() {- for (const url of this.seeds) this.frontier.add(url, 50, null, 'start page');+ for (const url of this.seeds) {+ const key = canonical(url);+ if (key) this.visited.delete(key);+ this.frontier.add(url, 50, null, 'start page');+ }if (!this.frontier.size) await this.searchForLeads();}@@ -178,10 +187,15 @@ export class Crawler {return urls.length;}+ // Called when pickNext() found nothing. Every path either adds an unvisited+ // lead or really waits, so run() can never spin without yielding.async reseedOrRest() {- const sinceReseed = Date.now() - this.lastReseedAt;- if (sinceReseed > 120_000) {- for (const url of this.seeds) if (!this.visited.has(canonical(url))) this.frontier.add(url, 50);+ for (const item of [...this.frontier.items.values()]) {+ if (this.visited.has(item.url)) this.frontier.drop(item.url);+ }+ if (Date.now() - this.lastReseedAt > 120_000) {+ this.lastReseedAt = Date.now();+ for (const url of this.seeds) if (!this.visited.has(canonical(url))) this.frontier.add(url, 50, null, 'start page');if (this.frontier.size || (await this.searchForLeads())) return;}this.status = 'resting';@@ -211,6 +225,20 @@ export class Crawler {return item;}+ // Two links from one page that both bounced back to it mean the rest of+ // its links will too (a hash-routed docs shell, a forced locale redirect):+ // drop them all so the agent stops walking in circles on one screen.+ noteBounce(home) {+ const n = (this.bounces.get(home) || 0) + 1;+ this.bounces.set(home, n);+ if (n !== 2) return;+ let dropped = 0;+ for (const item of [...this.frontier.items.values()]) {+ if (item.from === home) { this.frontier.drop(item.url); dropped += 1; }+ }+ if (dropped) this.log(`${home} redirects its links back to itself; dropped ${dropped} leads`);+ }+async pushStep(step, extra = {}) {this.seq += 1;return this.api.push({ agentId: this.agentId, step: { ...step, seq: this.seq }, ...extra });@@ -278,6 +306,13 @@ export class Crawler {await this.page.waitForLoadState('load', { timeout: 6000 }).catch(() => {});const finalUrl = canonical(this.page.url()) || url;+ // Redirected onto a page this agent already read (a docs shell that+ // bounces every deep link back to itself, a paywall home page): nothing+ // new to read here, so let the frontier choose without dwelling.+ if (finalUrl !== url && this.visited.has(finalUrl)) {+ this.noteBounce(finalUrl);+ return null;+ }this.visited.add(finalUrl);const finalHost = hostOf(finalUrl);this.domainCounts.set(finalHost, (this.domainCounts.get(finalHost) || 0) + 1);diff --git a/workers/agent-crawler/net.js b/workers/agent-crawler/guard.jssimilarity index 53%rename from workers/agent-crawler/net.jsrename to workers/agent-crawler/guard.jsindex b1974e0d5..3b9595103 100644--- a/workers/agent-crawler/net.js+++ b/workers/agent-crawler/guard.js@@ -1,16 +1,13 @@-// Network fences for a browser that goes wherever links lead.+// The network fence for a browser that goes wherever links lead.//// guardHost(): every request the crawler's pages make is resolved first and// refused when ANY address it resolves to is private, loopback, link-local or// reserved. That is what keeps a hostile page (or a mission seed pointing at an// internal name) from turning the crawler into a probe of the network it runs in.-//-// Robots: robots.txt is fetched once per origin and honored, including-// Crawl-delay, under the crawler's own product token.+// Dependency-free so tests/agent-crawler-guard.test.js can pin it.import dns from 'node:dns/promises';import net from 'node:net';-import robotsParser from 'robots-parser';export const BOT_TOKEN = 'three.ws-crawler';export const USER_AGENT =@@ -23,9 +20,11 @@ for (const [addr, prefix] of [['203.0.113.0', 24], ['224.0.0.0', 4], ['240.0.0.0', 4],]) blocked.addSubnet(addr, prefix, 'ipv4');for (const [addr, prefix] of [- ['::', 128], ['::1', 128], ['::ffff:0:0', 96], ['64:ff9b::', 96], ['100::', 64], ['2001:db8::', 32],- ['fc00::', 7], ['fe80::', 10], ['ff00::', 8],+ ['::', 128], ['::1', 128], ['100::', 64], ['2001:db8::', 32], ['fc00::', 7], ['fe80::', 10], ['ff00::', 8],]) blocked.addSubnet(addr, prefix, 'ipv6');+// No ::ffff:0:0/96 rule on purpose: BlockList already checks an IPv4-mapped+// address against the IPv4 rules above, and a mapped-range rule would match+// EVERY plain IPv4 address, refusing the whole public web.export function isBlockedAddress(ip) {const family = net.isIP(ip);@@ -66,51 +65,3 @@ export async function guardedFetch(url, { timeoutMs = 8000, headers = {} } = {})signal: AbortSignal.timeout(timeoutMs),});}--const robotsCache = new Map(); // origin -> { robots, at }-const ROBOTS_TTL_MS = 60 * 60_000;--async function robotsFor(origin) {- const hit = robotsCache.get(origin);- if (hit && Date.now() - hit.at < ROBOTS_TTL_MS) return hit.robots;- const robotsUrl = `${origin}/robots.txt`;- let body = '';- try {- const res = await guardedFetch(robotsUrl, { timeoutMs: 6000, headers: { accept: 'text/plain' } });- // RFC 9309: 4xx means no rules; 5xx or a network failure means "assume- // disallowed for now" so a struggling server is not crawled harder.- if (res.status >= 500) body = 'User-agent: *\nDisallow: /';- else if (res.ok) body = (await res.text()).slice(0, 500_000);- } catch {- body = 'User-agent: *\nDisallow: /';- }- const robots = robotsParser(robotsUrl, body);- robotsCache.set(origin, { robots, at: Date.now() });- if (robotsCache.size > 3000) robotsCache.delete(robotsCache.keys().next().value);- return robots;-}--// { allowed, delayMs } for one URL under the crawler's token.-export async function robotsCheck(url) {- const u = new URL(url);- const robots = await robotsFor(u.origin);- const allowed = robots.isAllowed(u.href, BOT_TOKEN) !== false;- const delay = Number(robots.getCrawlDelay(BOT_TOKEN));- return { allowed, delayMs: Number.isFinite(delay) && delay > 0 ? Math.min(delay, 30) * 1000 : 0 };-}--// Per-domain politeness shared by every crawler in the process: at most one-// request to a domain every gapMs (or its robots Crawl-delay, if longer).-const domainNext = new Map();--export function domainReadyIn(host) {- return Math.max(0, (domainNext.get(host) || 0) - Date.now());-}--export function claimDomain(host, gapMs) {- domainNext.set(host, Date.now() + gapMs);- if (domainNext.size > 10_000) {- const now = Date.now();- for (const [h, t] of domainNext) if (t < now) domainNext.delete(h);- }-}diff --git a/workers/agent-crawler/robots.js b/workers/agent-crawler/robots.jsnew file mode 100644index 000000000..c807e6ecc--- /dev/null+++ b/workers/agent-crawler/robots.js@@ -0,0 +1,53 @@+// robots.txt and per-domain politeness. robots.txt is fetched once per origin+// and honored, including Crawl-delay, under the crawler's own product token.++import robotsParser from 'robots-parser';+import { BOT_TOKEN, guardedFetch } from './guard.js';++const robotsCache = new Map(); // origin -> { robots, at }+const ROBOTS_TTL_MS = 60 * 60_000;++async function robotsFor(origin) {+ const hit = robotsCache.get(origin);+ if (hit && Date.now() - hit.at < ROBOTS_TTL_MS) return hit.robots;+ const robotsUrl = `${origin}/robots.txt`;+ let body = '';+ try {+ const res = await guardedFetch(robotsUrl, { timeoutMs: 6000, headers: { accept: 'text/plain' } });+ // RFC 9309: 4xx means no rules; 5xx or a network failure means "assume+ // disallowed for now" so a struggling server is not crawled harder.+ if (res.status >= 500) body = 'User-agent: *\nDisallow: /';+ else if (res.ok) body = (await res.text()).slice(0, 500_000);+ } catch {+ body = 'User-agent: *\nDisallow: /';+ }+ const robots = robotsParser(robotsUrl, body);+ robotsCache.set(origin, { robots, at: Date.now() });+ if (robotsCache.size > 3000) robotsCache.delete(robotsCache.keys().next().value);+ return robots;+}++// { allowed, delayMs } for one URL under the crawler's token.+export async function robotsCheck(url) {+ const u = new URL(url);+ const robots = await robotsFor(u.origin);+ const allowed = robots.isAllowed(u.href, BOT_TOKEN) !== false;+ const delay = Number(robots.getCrawlDelay(BOT_TOKEN));+ return { allowed, delayMs: Number.isFinite(delay) && delay > 0 ? Math.min(delay, 30) * 1000 : 0 };+}++// Per-domain politeness shared by every crawler in the process: at most one+// request to a domain every gapMs (or its robots Crawl-delay, if longer).+const domainNext = new Map();++export function domainReadyIn(host) {+ return Math.max(0, (domainNext.get(host) || 0) - Date.now());+}++export function claimDomain(host, gapMs) {+ domainNext.set(host, Date.now() + gapMs);+ if (domainNext.size > 10_000) {+ const now = Date.now();+ for (const [h, t] of domainNext) if (t < now) domainNext.delete(h);+ }+}diff --git a/workers/agent-crawler/seeds.js b/workers/agent-crawler/seeds.jsindex 341a2f181..3355c6470 100644--- a/workers/agent-crawler/seeds.js+++ b/workers/agent-crawler/seeds.js@@ -2,7 +2,7 @@// walked itself into a dead end: real public search, tried in order until one// answers. Each source is a free, keyless public endpoint.-import { guardedFetch } from './net.js';+import { guardedFetch } from './guard.js';// DuckDuckGo's HTML endpoint. Result anchors carry the target in the uddg// parameter of a /l/ redirect. It sometimes answers bots with a 202 challenge