Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// One agent out reading: a browser context of its own, a frontier of leads, and
// a loop that reads a page, chooses a link, walks to it on screen, and leaps.
//
// Every phase is pushed to the API as a live step (api/crawl/[action].js
// action=push) so /crawl can draw the agent's avatar walking across the real
// screenshot to the link it really chose. The page it finished reading rides
// along with the "reading" step and lands in the open corpus.
import { readPage, revealLink, visibleLinks, CONSENT_CSS } from './extract.js';
import {
Frontier, canonical, gistOf, hostOf, looksLikeWall, relevance, scoreLink, thoughtFor, topicTerms,
} from './frontier.js';
import { USER_AGENT, guardHost } from './guard.js';
import { claimDomain, domainReadyIn, robotsAllowsCached, robotsCheck } from './robots.js';
import { searchSeeds } from './seeds.js';
export const VIEWPORT = { width: 1200, height: 750 };
const STEP_LINKS = 48;
const MIN_TEXT_FOR_CORPUS = 200;
const MAX_SEARCHES_WITHOUT_READ = 3;
const FRAME_B64_MAX = 690_000;
const RECYCLE_PAGE_EVERY = 40;
export class NotEnrolledError extends Error {}
const sleep = (ms) => new Promise((r) => { setTimeout(r, ms); });
export class Crawler {
constructor({ mission, browser, api, log, opts }) {
this.agentId = mission.agentId;
this.name = mission.name;
this.topic = mission.topic;
this.seeds = mission.seeds || [];
this.terms = topicTerms(mission.topic);
this.browser = browser;
this.api = api;
this.opts = opts;
this.log = (msg) => log(`[${this.name}] ${msg}`);
this.frontier = new Frontier();
this.visited = new Set();
this.bounces = new Map(); // page -> links of it that redirected back to it
this.domainCounts = new Map();
this.seq = 0;
this.stopped = false;
this.startedAt = Date.now();
this.pagesRead = mission.pagesRead || 0;
this.navigations = 0;
this.currentUrl = null;
this.status = 'starting';
this.lastReseedAt = 0;
this.searchRound = 0;
this.searchesSinceRead = 0;
this.done = null;
}
summary() {
return {
agentId: this.agentId, name: this.name, topic: this.topic, status: this.status,
url: this.currentUrl, frontier: this.frontier.size, visited: this.visited.size,
pagesRead: this.pagesRead, upMs: Date.now() - this.startedAt,
};
}
updateMission(mission) {
if (mission.topic !== this.topic) {
this.topic = mission.topic;
this.terms = topicTerms(mission.topic);
this.frontier = new Frontier();
this.log(`topic changed to "${mission.topic}", starting fresh`);
}
this.seeds = mission.seeds || [];
}
start() {
this.done = this.run().catch((err) => {
if (!(err instanceof NotEnrolledError)) this.log(`stopped on error: ${err?.message || err}`);
}).finally(() => this.close());
return this.done;
}
async stop() {
this.stopped = true;
await this.close();
await this.done?.catch(() => {});
}
async close() {
const ctx = this.context;
this.context = null;
this.page = null;
if (ctx) await ctx.close().catch(() => {});
}
async openContext() {
this.context = await this.browser.newContext({
viewport: VIEWPORT,
deviceScaleFactor: 1,
userAgent: USER_AGENT,
locale: 'en-US',
acceptDownloads: false,
javaScriptEnabled: true,
serviceWorkers: 'block',
});
await this.context.addInitScript((css) => {
const add = () => {
if (document.getElementById('tws-crawl-consent')) return;
const s = document.createElement('style');
s.id = 'tws-crawl-consent';
s.textContent = css;
(document.head || document.documentElement).appendChild(s);
};
if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', add, { once: true });
else add();
}, CONSENT_CSS);
// Every request is fenced: private hosts are refused and media streams
// (which never show in a still frame) are skipped.
await this.context.route('**/*', async (route) => {
const req = route.request();
if (req.resourceType() === 'media') return route.abort('blockedbyclient');
let host = '';
try { host = new URL(req.url()).hostname; } catch { return route.abort('blockedbyclient'); }
if (req.url().startsWith('data:') || req.url().startsWith('blob:')) return route.continue();
if (!(await guardHost(host))) return route.abort('blockedbyclient');
return route.continue();
});
this.context.on('page', (p) => {
if (this.page && p !== this.page) p.close().catch(() => {});
});
await this.newPage();
}
async newPage() {
if (this.page) await this.page.close().catch(() => {});
this.page = await this.context.newPage();
this.page.on('dialog', (d) => d.dismiss().catch(() => {}));
this.navigations = 0;
}
async run() {
await this.openContext();
await this.loadHistory();
await this.seed();
// A walked-to link is visited straight away (the avatar is standing on
// it); otherwise the frontier picks the next lead.
let following = null;
while (!this.stopped) {
const next = following || this.pickNext();
following = null;
if (!next) {
await this.reseedOrRest();
continue;
}
try {
following = await this.visit(next);
} catch (err) {
if (err instanceof NotEnrolledError || this.stopped) throw err;
following = null;
await this.pageFault(next.url, err);
}
}
}
// One page misbehaving (a crash, a self-destructing script, a hung
// screenshot) costs that page, never the agent's shift: say so on screen,
// get a fresh tab if the old one is gone, and carry on with the frontier.
async pageFault(url, err) {
const msg = String(err?.message || err);
this.log(`${url} fell over (${msg.split('\n')[0]}); moving on`);
await this.reviveTab(msg);
await this.blocked(hostOf(url) || 'that site', 'fell over while it was being read');
}
// A crashed or closed tab fails every later navigation instantly, so an
// agent that keeps it would burn through its whole queue in seconds,
// filing each lead as blocked. Replace it whenever an error says so.
async reviveTab(msg) {
if (!this.page || this.page.isClosed() || /Target closed|crashed|has been closed/i.test(msg)) await this.newPage();
}
// Pages this agent already read survive restarts: the corpus is the memory.
async loadHistory() {
try {
const pages = await this.api.recentPages(this.agentId, 500);
for (const p of pages) {
const key = canonical(p.url);
if (key) this.visited.add(key);
const h = hostOf(p.url);
this.domainCounts.set(h, Math.min(10, (this.domainCounts.get(h) || 0) + 1));
}
} catch (err) {
this.log(`history unavailable (${err?.message || err}); starting with a clean slate`);
}
}
// Start pages are re-read on every shift even if the corpus already holds
// them: they are where the owner pointed the agent, and their links are the
// freshest leads it has. The API files a re-read page only once.
async seed() {
for (const url of this.seeds) {
const key = canonical(url);
if (key) this.visited.delete(key);
this.frontier.add(url, 50, null, 'start page');
}
if (!this.frontier.size) await this.searchForLeads();
}
async searchForLeads() {
this.lastReseedAt = Date.now();
this.searchesSinceRead += 1;
const { urls, source } = await searchSeeds(this.topic, {
log: this.log,
round: this.searchRound++,
fresh: (u) => !this.visited.has(canonical(u)),
});
urls.forEach((u, i) => this.frontier.add(u, 20 - i * 0.5, null, ''));
if (urls.length) this.log(`${urls.length} leads from ${source}`);
return urls.length;
}
// Called when pickNext() found nothing. Every path either adds an unvisited
// lead or really waits, so run() can never spin without yielding. Searching
// again straight away is fine while searches keep turning into read pages;
// MAX_SEARCHES_WITHOUT_READ dry rounds in a row fall back to the slow cadence.
async reseedOrRest() {
for (const item of [...this.frontier.items.values()]) {
if (this.visited.has(item.url)) this.frontier.drop(item.url);
}
const searchNow = this.searchesSinceRead < MAX_SEARCHES_WITHOUT_READ || Date.now() - this.lastReseedAt > 120_000;
if (searchNow) {
this.lastReseedAt = Date.now();
for (const url of this.seeds) if (!this.visited.has(canonical(url))) this.frontier.add(url, 50, null, 'start page');
if (this.frontier.size || (await this.searchForLeads())) return;
}
this.status = 'resting';
await this.pushStep({
url: this.currentUrl, title: '', status: 'resting', links: [], target: null,
thought: thoughtFor('resting', { reason: `Out of fresh leads on ${this.topic}. Searching again shortly.` }),
});
await this.idle(60_000);
}
async idle(ms) {
const until = Date.now() + ms;
while (!this.stopped && Date.now() < until) await sleep(Math.min(1000, until - Date.now()));
}
pickNext() {
const accept = (item) => {
if (this.visited.has(item.url)) return false;
return domainReadyIn(hostOf(item.url)) <= 2500;
};
let item = this.frontier.pop(accept);
if (!item && this.frontier.size) {
// Everything left is on a cooling domain: take the best one; visit()
// waits out its politeness gap before navigating.
item = this.frontier.pop((i) => !this.visited.has(i.url));
}
return item;
}
// A site refused one page: drop every queued link on that host its
// robots.txt also refuses, so the agent does not burn through them one
// refusal at a time (an unreachable site refuses all of them).
purgeRefused(host) {
let dropped = 0;
for (const item of [...this.frontier.items.values()]) {
if (hostOf(item.url) === host && robotsAllowsCached(item.url) === false) {
this.frontier.drop(item.url);
dropped += 1;
}
}
if (dropped) this.log(`${host} refuses crawlers on ${dropped} more queued links; dropped them`);
}
// Two links from one page that both bounced back to it mean the rest of
// its links will too (a hash-routed docs shell, a forced locale redirect):
// drop them all so the agent stops walking in circles on one screen.
noteBounce(home) {
const n = (this.bounces.get(home) || 0) + 1;
this.bounces.set(home, n);
if (n !== 2) return;
let dropped = 0;
for (const item of [...this.frontier.items.values()]) {
if (item.from === home) { this.frontier.drop(item.url); dropped += 1; }
}
if (dropped) this.log(`${home} redirects its links back to itself; dropped ${dropped} leads`);
}
async pushStep(step, extra = {}) {
this.seq += 1;
return this.api.push({ agentId: this.agentId, step: { ...step, seq: this.seq }, ...extra });
}
// Some pages replace themselves right after load (a client-side redirect, a
// consent wall reloading the document), which destroys the context mid-read.
// Wait for the replacement to land and read that instead, once.
async readSettled() {
try {
return await this.page.evaluate(readPage, 600);
} catch (err) {
if (!/Execution context was destroyed|navigat/i.test(String(err?.message || err))) throw err;
await this.page.waitForLoadState('load', { timeout: 6000 }).catch(() => {});
const settled = canonical(this.page.url());
if (settled) {
this.visited.add(settled);
this.currentUrl = settled;
}
return this.page.evaluate(readPage, 600);
}
}
async screenshot() {
let quality = this.opts.jpegQuality;
for (let attempt = 0; attempt < 3; attempt++) {
const buf = await this.page.screenshot({ type: 'jpeg', quality, timeout: 8000, animations: 'disabled', caret: 'hide' });
const b64 = buf.toString('base64');
if (b64.length <= FRAME_B64_MAX) return b64;
quality = Math.max(25, quality - 15);
}
return null;
}
async blocked(host, reason) {
this.status = 'blocked';
await this.pushStep({
url: this.currentUrl, title: '', status: 'blocked', links: [], target: null,
thought: thoughtFor('blocked', { host, reason }),
}).catch((err) => { if (err instanceof NotEnrolledError) throw err; });
}
// Read one page, then decide where to go. Returns the frontier entry to visit
// next when the agent chose an on-screen link (it already walked there), or
// null to let the frontier decide.
async visit(target) {
const url = canonical(target.url);
if (!url) return null;
const host = hostOf(url);
this.visited.add(url);
const robots = await robotsCheck(url).catch(() => ({ allowed: false, delayMs: 0 }));
if (!robots.allowed) {
this.purgeRefused(host);
await this.blocked(host, 'asks crawlers to stay out of that page');
return null;
}
const wait = domainReadyIn(host);
if (wait > 0) await sleep(wait);
claimDomain(host, Math.max(this.opts.domainGapMs, robots.delayMs));
if (this.navigations >= RECYCLE_PAGE_EVERY) await this.newPage();
this.navigations += 1;
let resp;
try {
resp = await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.opts.navTimeoutMs });
} catch (err) {
const msg = String(err?.message || err);
const refused = /ERR_BLOCKED_BY_CLIENT/.test(msg);
if (!refused) this.log(`${url} did not load (${msg.split('\n')[0]})`);
await this.reviveTab(msg);
await this.blocked(host, refused ? 'is not a public address' : 'did not answer in time');
return null;
}
if (!resp || resp.status() >= 400) {
await this.blocked(host, `answered ${resp ? resp.status() : 'nothing'}`);
return null;
}
const ctype = resp.headers()['content-type'] || '';
if (ctype && !/html|xhtml/i.test(ctype)) {
await this.blocked(host, 'sent a file, not a page');
return null;
}
// Let the page paint its above-the-fold content before the frame is taken.
await this.page.waitForLoadState('load', { timeout: 6000 }).catch(() => {});
let finalUrl = canonical(this.page.url()) || url;
// Redirected onto a page this agent already read (a docs shell that
// bounces every deep link back to itself, a paywall home page): nothing
// new to read here, so let the frontier choose without dwelling.
if (finalUrl !== url && this.visited.has(finalUrl)) {
this.noteBounce(finalUrl);
return null;
}
this.visited.add(finalUrl);
let finalHost = hostOf(finalUrl);
this.domainCounts.set(finalHost, (this.domainCounts.get(finalHost) || 0) + 1);
this.currentUrl = finalUrl;
const data = await this.readSettled();
if (this.currentUrl !== finalUrl) {
finalUrl = this.currentUrl;
finalHost = hostOf(finalUrl);
}
if (looksLikeWall(data.title, data.text)) {
await this.blocked(finalHost, 'put up a wall instead of the page');
return null;
}
const rel = relevance(this.terms, data.title, data.text);
const gist = gistOf(data.text, this.terms);
// Score every link on the page into the frontier. Links found on a
// relevant page inherit some of its relevance (topical locality).
const ctx = { terms: this.terms, fromHost: finalHost, domainCounts: this.domainCounts, visited: this.visited, rand: Math.random };
let best = null;
for (const link of data.links) {
const s = scoreLink(link, ctx);
if (!Number.isFinite(s)) continue;
const score = s + rel * 1.5;
this.frontier.add(link.href, score, finalUrl, link.text);
if (!best || score > best.score) best = { ...link, score };
}
this.frontier.decay();
const frame = await this.screenshot();
const shown = data.links.filter((l) => l.visible).slice(0, STEP_LINKS).map(({ x, y, w, h, text }) => ({ x, y, w, h, t: text }));
const page = data.text.length >= MIN_TEXT_FOR_CORPUS ? {
url: finalUrl, title: data.title, text: data.text, gist, linksOut: data.links.length, relevance: rel, fromUrl: target.from,
} : null;
this.status = 'reading';
const res = await this.pushStep({
url: finalUrl, title: data.title, status: 'reading', links: shown, target: null, scrollY: data.scrollY,
thought: thoughtFor('reading', { title: data.title || finalHost, domain: finalHost, topic: this.topic, relevance: rel }),
}, { frame, page });
if (res?.recorded) {
this.pagesRead = res.pagesRead || this.pagesRead + 1;
this.searchesSinceRead = 0;
}
// Read: dwell on the page roughly in proportion to how much there is.
await this.idle(this.opts.readMs + Math.min(3000, data.text.length / 20));
if (this.stopped) return null;
const frontierBest = [...this.frontier.items.values()]
.filter((i) => !this.visited.has(i.url) && i.url !== canonical(best?.href))
.reduce((m, i) => (!m || i.score > m.score ? i : m), null);
if (best && (!frontierBest || best.score >= frontierBest.score - 0.5)) {
const walked = await this.walkTo(best, finalUrl, finalHost);
if (walked) return walked;
}
// Leap: the best lead is not on this screen.
const lead = frontierBest;
if (lead) {
this.status = 'leaping';
await this.pushStep({
url: finalUrl, title: data.title, status: 'leaping', links: [], target: null,
thought: thoughtFor('leaping', { text: lead.text, host: hostOf(lead.url) }),
});
await this.idle(this.opts.leapMs);
}
return null;
}
// Bring the chosen link on screen, push the walking step with its rect as the
// target, give the avatar time to walk there, then hop.
async walkTo(link, pageUrl, pageHost) {
const ok = await this.page.evaluate(revealLink, link.i).catch(() => false);
if (!ok) return null;
const measured = await this.page.evaluate(visibleLinks, 400).catch(() => null);
if (!measured) return null;
let list = measured.links;
let idx = list.findIndex((l) => l.i === link.i);
if (idx < 0) return null;
if (list.length > STEP_LINKS) {
const tgt = list[idx];
list = [tgt, ...list.filter((l) => l !== tgt).slice(0, STEP_LINKS - 1)];
idx = 0;
}
const frame = link.visible ? null : await this.screenshot();
const host = hostOf(link.href);
this.status = 'walking';
await this.pushStep({
url: pageUrl, title: '', status: 'walking', scrollY: measured.scrollY,
links: list.map(({ x, y, w, h, t }) => ({ x, y, w, h, t })), target: idx, nextUrl: link.href,
thought: thoughtFor('walking', { text: link.text, host, offsite: host !== pageHost }),
}, frame ? { frame } : {});
await this.idle(this.opts.walkMs);
if (this.stopped) return null;
await this.pushStep({
url: pageUrl, title: '', status: 'leaping', scrollY: measured.scrollY,
links: list.map(({ x, y, w, h, t }) => ({ x, y, w, h, t })), target: idx, nextUrl: link.href,
thought: thoughtFor('opening', { text: link.text, host, offsite: host !== pageHost }),
});
await this.idle(this.opts.leapMs);
this.frontier.drop(link.href);
return { url: link.href, score: link.score, from: pageUrl, text: link.text };
}
}