Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(crawl): keep crawlers reading through dead tabs, bot walls and exhausted searches
nirholasf72d40cd2c4e
diff --git a/api/crawl/[action].js b/api/crawl/[action].jsindex e44e5ce66..c28511bd4 100644--- a/api/crawl/[action].js+++ b/api/crawl/[action].js@@ -220,6 +220,10 @@ async function handlePush(req, res) {r.zremrangebyscore(ACTIVE_KEY, 0, now - CRAWL_ACTIVE_WINDOW_MS * 2),];if (frame) writes.push(r.set(FRAME_KEY(agentId), frame, { ex: CRAWL_LIVE_TTL }));+ // The live record keeps pointing at the last frame, so that frame must live+ // exactly as long as the record does, or a crawler that rests on one page+ // for longer than the TTL serves a frameTs whose image is gone.+ else if (live.frameTs) writes.push(r.expire(FRAME_KEY(agentId), CRAWL_LIVE_TTL));if (recorded?.inserted) {const entry = { agentId, name: mission.name, url: page.url, title: page.title, domain: page.domain, tokens: page.tokens, gist: page.gist, ts: now };writes.push(r.lpush(FEED_KEY, JSON.stringify(entry)).then(() => r.ltrim(FEED_KEY, 0, FEED_CAP - 1)));diff --git a/tests/agent-crawler-frontier.test.js b/tests/agent-crawler-frontier.test.jsindex e2b5e7fdd..e4d14eb61 100644--- a/tests/agent-crawler-frontier.test.js+++ b/tests/agent-crawler-frontier.test.js@@ -2,7 +2,7 @@// the rules that decide where an agent walks are pinned here.import { describe, expect, it } from 'vitest';import {- Frontier, MAX_PAGES_PER_DOMAIN, canonical, gistOf, isCrawlable, relevance, scoreLink, stem, thoughtFor, tokenize, topicTerms,+ Frontier, MAX_PAGES_PER_DOMAIN, canonical, gistOf, isCrawlable, looksLikeWall, relevance, scoreLink, stem, thoughtFor, tokenize, topicTerms,} from '../workers/agent-crawler/frontier.js';const ctx = (over = {}) => ({ terms: topicTerms('solana validator economics'), fromHost: 'example.com', domainCounts: new Map(), visited: new Set(), ...over });@@ -104,6 +104,21 @@ describe('thoughtFor', () => {it('narrates what the agent is really doing', () => {expect(thoughtFor('reading', { title: 'Rewards', domain: 'e.com', topic: 'validators', relevance: 0.5 })).toMatch(/squarely about validators/);expect(thoughtFor('walking', { text: 'Next page', host: 'b.org', offsite: true })).toBe('Walking to "Next page", off to b.org.');+ expect(thoughtFor('opening', { text: 'Next page' })).toBe('Opening "Next page".');expect(thoughtFor('blocked', { host: 'e.com', reason: 'answered 403' })).toBe('e.com answered 403. Finding another way.');});});++describe('looksLikeWall', () => {+ it('catches refusals, bot challenges and soft 404s that answered 200', () => {+ expect(looksLikeWall('Access Denied', 'You don\'t have permission to access this server.')).toBe(true);+ expect(looksLikeWall('Just a moment...', 'Checking your browser before accessing the site.')).toBe(true);+ expect(looksLikeWall('Example', 'Page not found. The page you asked for moved.')).toBe(true);+ });++ it('still reads a real article that mentions a wall', () => {+ const article = 'Access denied errors on validators usually mean a misconfigured RPC allowlist. '.repeat(30);+ expect(looksLikeWall('Debugging RPC access', article)).toBe(false);+ expect(looksLikeWall('Validator rewards explained', 'Rewards come from inflation and fees.')).toBe(false);+ });+});diff --git a/tests/agent-crawler-seeds.test.js b/tests/agent-crawler-seeds.test.jsnew file mode 100644index 000000000..fce6e4780--- /dev/null+++ b/tests/agent-crawler-seeds.test.js@@ -0,0 +1,35 @@+// The Crawl worker's lead search (workers/agent-crawler/seeds.js): a source only+// counts when it returns pages the agent has not read, and sources are pooled+// until there are enough, so a long-running agent never rests on stale hits.+import { afterEach, describe, expect, it } from 'vitest';+import { SOURCES, searchSeeds } from '../workers/agent-crawler/seeds.js';++const real = [...SOURCES];+const use = (...fns) => SOURCES.splice(0, SOURCES.length, ...fns.map((fn, i) => ({ name: `s${i}`, fn })));+afterEach(() => SOURCES.splice(0, SOURCES.length, ...real));++describe('searchSeeds', () => {+ it('skips a source whose hits were all read and pools the next ones', async () => {+ const read = new Set(['https://a.example/1', 'https://a.example/2']);+ use(async () => [...read], async () => ['https://b.example/1', 'https://b.example/2'], async () => ['https://c.example/1']);+ const { urls, source } = await searchSeeds('topic', { fresh: (u) => !read.has(u) });+ expect(urls).toEqual(['https://b.example/1', 'https://b.example/2', 'https://c.example/1']);+ expect(source).toBe('s1 and s2');+ });++ it('stops asking once it has enough and passes the round through', async () => {+ const rounds = [];+ const many = Array.from({ length: 8 }, (_, i) => `https://a.example/${i}`);+ use(async (_t, round) => { rounds.push(round); return many; }, async () => { throw new Error('should not be asked'); });+ const { urls } = await searchSeeds('topic', { round: 3 });+ expect(urls).toHaveLength(8);+ expect(rounds).toEqual([3]);+ });++ it('survives a failing source and reports nothing found honestly', async () => {+ const logs = [];+ use(async () => { throw new Error('down'); }, async () => []);+ expect(await searchSeeds('topic', { log: (m) => logs.push(m) })).toEqual({ urls: [], source: null });+ expect(logs[0]).toMatch(/s0 failed: down/);+ });+});diff --git a/workers/agent-crawler/crawler.js b/workers/agent-crawler/crawler.jsindex 211a0aa2e..dd9cd2fe7 100644--- a/workers/agent-crawler/crawler.js+++ b/workers/agent-crawler/crawler.js@@ -8,15 +8,16 @@import { readPage, revealLink, visibleLinks, CONSENT_CSS } from './extract.js';import {- Frontier, canonical, gistOf, hostOf, relevance, scoreLink, thoughtFor, topicTerms,+ Frontier, canonical, gistOf, hostOf, looksLikeWall, relevance, scoreLink, thoughtFor, topicTerms,} from './frontier.js';import { USER_AGENT, guardHost } from './guard.js';-import { claimDomain, domainReadyIn, robotsCheck } from './robots.js';+import { claimDomain, domainReadyIn, robotsAllowsCached, robotsCheck } from './robots.js';import { searchSeeds } from './seeds.js';export const VIEWPORT = { width: 1200, height: 750 };const STEP_LINKS = 48;const MIN_TEXT_FOR_CORPUS = 200;+const MAX_SEARCHES_WITHOUT_READ = 3;const FRAME_B64_MAX = 690_000;const RECYCLE_PAGE_EVERY = 40;@@ -47,6 +48,8 @@ export class Crawler {this.currentUrl = null;this.status = 'starting';this.lastReseedAt = 0;+ this.searchRound = 0;+ this.searchesSinceRead = 0;this.done = null;}@@ -148,10 +151,33 @@ export class Crawler {await this.reseedOrRest();continue;}- following = await this.visit(next);+ try {+ following = await this.visit(next);+ } catch (err) {+ if (err instanceof NotEnrolledError || this.stopped) throw err;+ following = null;+ await this.pageFault(next.url, err);+ }}}+ // One page misbehaving (a crash, a self-destructing script, a hung+ // screenshot) costs that page, never the agent's shift: say so on screen,+ // get a fresh tab if the old one is gone, and carry on with the frontier.+ async pageFault(url, err) {+ const msg = String(err?.message || err);+ this.log(`${url} fell over (${msg.split('\n')[0]}); moving on`);+ await this.reviveTab(msg);+ await this.blocked(hostOf(url) || 'that site', 'fell over while it was being read');+ }++ // A crashed or closed tab fails every later navigation instantly, so an+ // agent that keeps it would burn through its whole queue in seconds,+ // filing each lead as blocked. Replace it whenever an error says so.+ async reviveTab(msg) {+ if (!this.page || this.page.isClosed() || /Target closed|crashed|has been closed/i.test(msg)) await this.newPage();+ }+// Pages this agent already read survive restarts: the corpus is the memory.async loadHistory() {try {@@ -181,19 +207,27 @@ export class Crawler {async searchForLeads() {this.lastReseedAt = Date.now();- const { urls, source } = await searchSeeds(this.topic, this.log);+ this.searchesSinceRead += 1;+ const { urls, source } = await searchSeeds(this.topic, {+ log: this.log,+ round: this.searchRound++,+ fresh: (u) => !this.visited.has(canonical(u)),+ });urls.forEach((u, i) => this.frontier.add(u, 20 - i * 0.5, null, ''));if (urls.length) this.log(`${urls.length} leads from ${source}`);return urls.length;}// Called when pickNext() found nothing. Every path either adds an unvisited- // lead or really waits, so run() can never spin without yielding.+ // lead or really waits, so run() can never spin without yielding. Searching+ // again straight away is fine while searches keep turning into read pages;+ // MAX_SEARCHES_WITHOUT_READ dry rounds in a row fall back to the slow cadence.async reseedOrRest() {for (const item of [...this.frontier.items.values()]) {if (this.visited.has(item.url)) this.frontier.drop(item.url);}- if (Date.now() - this.lastReseedAt > 120_000) {+ const searchNow = this.searchesSinceRead < MAX_SEARCHES_WITHOUT_READ || Date.now() - this.lastReseedAt > 120_000;+ if (searchNow) {this.lastReseedAt = Date.now();for (const url of this.seeds) if (!this.visited.has(canonical(url))) this.frontier.add(url, 50, null, 'start page');if (this.frontier.size || (await this.searchForLeads())) return;@@ -225,6 +259,20 @@ export class Crawler {return item;}+ // A site refused one page: drop every queued link on that host its+ // robots.txt also refuses, so the agent does not burn through them one+ // refusal at a time (an unreachable site refuses all of them).+ purgeRefused(host) {+ let dropped = 0;+ for (const item of [...this.frontier.items.values()]) {+ if (hostOf(item.url) === host && robotsAllowsCached(item.url) === false) {+ this.frontier.drop(item.url);+ dropped += 1;+ }+ }+ if (dropped) this.log(`${host} refuses crawlers on ${dropped} more queued links; dropped them`);+ }+// Two links from one page that both bounced back to it mean the rest of// its links will too (a hash-routed docs shell, a forced locale redirect):// drop them all so the agent stops walking in circles on one screen.@@ -244,6 +292,24 @@ export class Crawler {return this.api.push({ agentId: this.agentId, step: { ...step, seq: this.seq }, ...extra });}+ // Some pages replace themselves right after load (a client-side redirect, a+ // consent wall reloading the document), which destroys the context mid-read.+ // Wait for the replacement to land and read that instead, once.+ async readSettled() {+ try {+ return await this.page.evaluate(readPage, 600);+ } catch (err) {+ if (!/Execution context was destroyed|navigat/i.test(String(err?.message || err))) throw err;+ await this.page.waitForLoadState('load', { timeout: 6000 }).catch(() => {});+ const settled = canonical(this.page.url());+ if (settled) {+ this.visited.add(settled);+ this.currentUrl = settled;+ }+ return this.page.evaluate(readPage, 600);+ }+ }+async screenshot() {let quality = this.opts.jpegQuality;for (let attempt = 0; attempt < 3; attempt++) {@@ -274,6 +340,7 @@ export class Crawler {const robots = await robotsCheck(url).catch(() => ({ allowed: false, delayMs: 0 }));if (!robots.allowed) {+ this.purgeRefused(host);await this.blocked(host, 'asks crawlers to stay out of that page');return null;}@@ -289,8 +356,10 @@ export class Crawler {resp = await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.opts.navTimeoutMs });} catch (err) {const msg = String(err?.message || err);- await this.blocked(host, /ERR_BLOCKED_BY_CLIENT/.test(msg) ? 'is not a public address' : 'did not answer in time');- if (/Target closed|crashed/i.test(msg)) await this.newPage();+ const refused = /ERR_BLOCKED_BY_CLIENT/.test(msg);+ if (!refused) this.log(`${url} did not load (${msg.split('\n')[0]})`);+ await this.reviveTab(msg);+ await this.blocked(host, refused ? 'is not a public address' : 'did not answer in time');return null;}if (!resp || resp.status() >= 400) {@@ -305,7 +374,7 @@ export class Crawler {// Let the page paint its above-the-fold content before the frame is taken.await this.page.waitForLoadState('load', { timeout: 6000 }).catch(() => {});- const finalUrl = canonical(this.page.url()) || url;+ let finalUrl = canonical(this.page.url()) || url;// Redirected onto a page this agent already read (a docs shell that// bounces every deep link back to itself, a paywall home page): nothing// new to read here, so let the frontier choose without dwelling.@@ -314,11 +383,19 @@ export class Crawler {return null;}this.visited.add(finalUrl);- const finalHost = hostOf(finalUrl);+ let finalHost = hostOf(finalUrl);this.domainCounts.set(finalHost, (this.domainCounts.get(finalHost) || 0) + 1);this.currentUrl = finalUrl;- const data = await this.page.evaluate(readPage, 600);+ const data = await this.readSettled();+ if (this.currentUrl !== finalUrl) {+ finalUrl = this.currentUrl;+ finalHost = hostOf(finalUrl);+ }+ if (looksLikeWall(data.title, data.text)) {+ await this.blocked(finalHost, 'put up a wall instead of the page');+ return null;+ }const rel = relevance(this.terms, data.title, data.text);const gist = gistOf(data.text, this.terms);@@ -346,7 +423,10 @@ export class Crawler {url: finalUrl, title: data.title, status: 'reading', links: shown, target: null, scrollY: data.scrollY,thought: thoughtFor('reading', { title: data.title || finalHost, domain: finalHost, topic: this.topic, relevance: rel }),}, { frame, page });- if (res?.recorded) this.pagesRead = res.pagesRead || this.pagesRead + 1;+ if (res?.recorded) {+ this.pagesRead = res.pagesRead || this.pagesRead + 1;+ this.searchesSinceRead = 0;+ }// Read: dwell on the page roughly in proportion to how much there is.await this.idle(this.opts.readMs + Math.min(3000, data.text.length / 20));@@ -401,7 +481,7 @@ export class Crawler {await this.pushStep({url: pageUrl, title: '', status: 'leaping', scrollY: measured.scrollY,links: list.map(({ x, y, w, h, t }) => ({ x, y, w, h, t })), target: idx, nextUrl: link.href,- thought: thoughtFor('walking', { text: link.text, host, offsite: host !== pageHost }),+ thought: thoughtFor('opening', { text: link.text, host, offsite: host !== pageHost }),});await this.idle(this.opts.leapMs);this.frontier.drop(link.href);diff --git a/workers/agent-crawler/frontier.js b/workers/agent-crawler/frontier.jsindex b751fd92b..396ebaca8 100644--- a/workers/agent-crawler/frontier.js+++ b/workers/agent-crawler/frontier.js@@ -187,6 +187,19 @@ export class Frontier {// Extractive gist: the first few real sentences, preferring ones that mention// the topic. No model call, so it is free, deterministic, and never invents.+// Pages that answered 200 but are a wall, not content: a CDN refusal, a bot+// challenge, a soft 404. Judged on the title plus the opening text, and only+// when the page is short, so an article that merely mentions "access denied"+// is still read.+const WALL = /\b(access denied|forbidden|attention required|just a moment|are you a (robot|human)|verify(ing)? you are (a )?human|checking your browser|request blocked|page not found|404 not found|not acceptable|too many requests|enable (javascript|cookies) to continue)\b/i;+const WALL_MAX_TEXT = 1500;++export function looksLikeWall(title, text) {+ const body = String(text || '');+ if (body.length > WALL_MAX_TEXT) return false;+ return WALL.test(`${title || ''} ${body.slice(0, 300)}`);+}+export function gistOf(text, terms, max = 480) {// Lines are block boundaries (readPage puts one per paragraph), so a menu// crumb on its own line never glues onto the first real sentence.@@ -229,6 +242,8 @@ export function thoughtFor(kind, d = {}) {return d.text? `Walking to "${quote(d.text, 60)}"${d.offsite ? `, off to ${d.host}` : ''}.`: `Walking to a link${d.offsite ? ` on ${d.host}` : ''}.`;+ case 'opening':+ return d.text ? `Opening "${quote(d.text, 60)}".` : `Opening the link${d.offsite ? ` on ${d.host}` : ''}.`;case 'leaping':return d.text ? `Leaping to "${quote(d.text, 60)}" from my list.` : `Leaping to ${d.host} from my list.`;case 'blocked':diff --git a/workers/agent-crawler/robots.js b/workers/agent-crawler/robots.jsindex c807e6ecc..e0e84aacb 100644--- a/workers/agent-crawler/robots.js+++ b/workers/agent-crawler/robots.js@@ -4,25 +4,29 @@import robotsParser from 'robots-parser';import { BOT_TOKEN, guardedFetch } from './guard.js';-const robotsCache = new Map(); // origin -> { robots, at }+const robotsCache = new Map(); // origin -> { robots, at, ttl }const ROBOTS_TTL_MS = 60 * 60_000;+// An unreachable or failing server is retried sooner than a real answer.+const ROBOTS_FAIL_TTL_MS = 10 * 60_000;async function robotsFor(origin) {const hit = robotsCache.get(origin);- if (hit && Date.now() - hit.at < ROBOTS_TTL_MS) return hit.robots;+ if (hit && Date.now() - hit.at < hit.ttl) return hit.robots;const robotsUrl = `${origin}/robots.txt`;let body = '';+ let ttl = ROBOTS_TTL_MS;try {const res = await guardedFetch(robotsUrl, { timeoutMs: 6000, headers: { accept: 'text/plain' } });// RFC 9309: 4xx means no rules; 5xx or a network failure means "assume// disallowed for now" so a struggling server is not crawled harder.- if (res.status >= 500) body = 'User-agent: *\nDisallow: /';+ if (res.status >= 500) { body = 'User-agent: *\nDisallow: /'; ttl = ROBOTS_FAIL_TTL_MS; }else if (res.ok) body = (await res.text()).slice(0, 500_000);} catch {body = 'User-agent: *\nDisallow: /';+ ttl = ROBOTS_FAIL_TTL_MS;}const robots = robotsParser(robotsUrl, body);- robotsCache.set(origin, { robots, at: Date.now() });+ robotsCache.set(origin, { robots, at: Date.now(), ttl });if (robotsCache.size > 3000) robotsCache.delete(robotsCache.keys().next().value);return robots;}@@ -36,6 +40,18 @@ export async function robotsCheck(url) {return { allowed, delayMs: Number.isFinite(delay) && delay > 0 ? Math.min(delay, 30) * 1000 : 0 };}+// Synchronous answer from the cache only: false when a cached robots.txt+// refuses the URL, true when it allows it, null when the origin is unknown.+// Lets the crawler purge every queued link a site already refused without+// visiting them one by one.+export function robotsAllowsCached(url) {+ let u;+ try { u = new URL(url); } catch { return null; }+ const hit = robotsCache.get(u.origin);+ if (!hit || Date.now() - hit.at >= hit.ttl) return null;+ return hit.robots.isAllowed(u.href, BOT_TOKEN) !== false;+}+// Per-domain politeness shared by every crawler in the process: at most one// request to a domain every gapMs (or its robots Crawl-delay, if longer).const domainNext = new Map();diff --git a/workers/agent-crawler/seeds.js b/workers/agent-crawler/seeds.jsindex 3355c6470..efa44eb58 100644--- a/workers/agent-crawler/seeds.js+++ b/workers/agent-crawler/seeds.js@@ -1,14 +1,16 @@// Where an agent starts when its owner gave no start pages, or when it has-// walked itself into a dead end: real public search, tried in order until one-// answers. Each source is a free, keyless public endpoint.+// walked itself into a dead end: real public search, tried in order until the+// agent has enough leads it has not read yet. Each source is a free, keyless+// public endpoint, and each one pages deeper on every round so a long-running+// agent keeps finding new ground instead of being handed the same top hits.import { guardedFetch } from './guard.js';// DuckDuckGo's HTML endpoint. Result anchors carry the target in the uddg// parameter of a /l/ redirect. It sometimes answers bots with a 202 challenge// page; that simply yields nothing and the next source takes over.-async function duckduckgo(topic) {- const res = await guardedFetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(topic)}`, {+async function duckduckgo(topic, round) {+ const res = await guardedFetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(topic)}&s=${round * 30}`, {headers: { accept: 'text/html' },});if (res.status !== 200) return [];@@ -26,10 +28,12 @@ async function duckduckgo(topic) {}// Hacker News search (Algolia): stories carry the outside URL they link to,-// which is exactly the kind of page a reading agent should land on.-async function hackernews(topic) {+// which is exactly the kind of page a reading agent should land on. Every word+// is optional, so a long topic still matches stories that share most of it.+async function hackernews(topic, round) {+ const q = encodeURIComponent(topic);const res = await guardedFetch(- `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(topic)}&tags=story&hitsPerPage=20`,+ `https://hn.algolia.com/api/v1/search?query=${q}&optionalWords=${q}&tags=story&hitsPerPage=20&page=${round}`,{ headers: { accept: 'application/json' } },);if (!res.ok) return [];@@ -38,9 +42,9 @@ async function hackernews(topic) {}// Wikipedia full-text search: always answers, always has something.-async function wikipedia(topic) {+async function wikipedia(topic, round) {const res = await guardedFetch(- `https://en.wikipedia.org/w/api.php?action=query&list=search&format=json&srlimit=8&srsearch=${encodeURIComponent(topic)}`,+ `https://en.wikipedia.org/w/api.php?action=query&list=search&format=json&srlimit=12&sroffset=${round * 12}&srsearch=${encodeURIComponent(topic)}`,{ headers: { accept: 'application/json' } },);if (!res.ok) return [];@@ -54,16 +58,26 @@ export const SOURCES = [{ name: 'wikipedia', fn: wikipedia },];-// Gather start pages for a topic. Returns { urls, source } from the first source-// with results, so the crawler can say where its leads came from.-export async function searchSeeds(topic, log = () => {}) {+const ENOUGH = 6;++// Gather leads for a topic. `fresh(url)` says whether the agent still wants a+// URL (it has not read it); sources are asked in order and their fresh results+// pooled until there are ENOUGH. Returns { urls, source } so the crawler can+// say where its leads came from.+export async function searchSeeds(topic, { log = () => {}, fresh = () => true, round = 0 } = {}) {+ const urls = [];+ const used = [];for (const { name, fn } of SOURCES) {try {- const urls = [...new Set(await fn(topic))].slice(0, 12);- if (urls.length) return { urls, source: name };+ const found = (await fn(topic, round)).filter((u) => !urls.includes(u) && fresh(u));+ if (found.length) {+ urls.push(...found);+ used.push(name);+ }} catch (err) {log(`seed source ${name} failed: ${err?.message || err}`);}+ if (urls.length >= ENOUGH) break;}- return { urls: [], source: null };+ return { urls: urls.slice(0, 12), source: used.join(' and ') || null };}