Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(crawl): add the crawl API, its schema and the agent-crawler worker behind the coming /crawl page

nirholas07eb02034bc2
diff --git a/api/_lib/crawl.js b/api/_lib/crawl.js
new file mode 100644
index 000000000..61623c221
--- /dev/null
+++ b/api/_lib/crawl.js
@@ -0,0 +1,182 @@
+// The Crawl: pure helpers shared by the API (api/crawl/[action].js), the worker
+// (workers/agent-crawler) and the tests. Nothing here touches the network, the
+// database or Redis, so every boundary rule (what a pushed step may carry, what a
+// mission may name, how a URL is keyed) is unit-testable and identical on both
+// sides of the wire.
+
+import { createHash } from 'node:crypto';
+
+export const CRAWL_LIVE_TTL = 75; // seconds a crawler stays "awake" after its last push
+export const CRAWL_ACTIVE_WINDOW_MS = 90_000;
+export const MAX_LINKS = 48;
+export const MAX_TEXT_CHARS = 120_000;
+export const GIST_MAX = 480;
+export const TITLE_MAX = 200;
+export const THOUGHT_MAX = 240;
+export const TOPIC_MIN = 2;
+export const TOPIC_MAX = 120;
+export const MAX_SEEDS = 8;
+export const FRAME_B64_MAX = 700_000; // ~520 KB JPEG
+export const CHARS_PER_TOKEN = 4;
+
+export const STEP_STATUSES = ['reading', 'walking', 'leaping', 'blocked', 'resting'];
+
+// Tracking parameters that make the same page look like many. Stripped before a
+// URL is hashed so the corpus dedupes on content, not on campaign tags.
+const TRACKING_PARAMS = /^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_[a-z]+|ref|ref_src|_hs[a-z]+|igshid|si)$/i;
+
+export function normalizeUrl(raw) {
+ let u;
+ try { u = new URL(String(raw)); } catch { return null; }
+ if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
+ if (u.username || u.password) return null;
+ u.hash = '';
+ u.hostname = u.hostname.toLowerCase().replace(/\.$/, '');
+ if ((u.protocol === 'https:' && u.port === '443') || (u.protocol === 'http:' && u.port === '80')) u.port = '';
+ for (const k of [...u.searchParams.keys()]) {
+ if (TRACKING_PARAMS.test(k)) u.searchParams.delete(k);
+ }
+ u.searchParams.sort();
+ if (u.pathname.length > 1 && u.pathname.endsWith('/')) u.pathname = u.pathname.replace(/\/+$/, '');
+ return u.href;
+}
+
+export function urlHash(normalized) {
+ return createHash('sha256').update(normalized).digest('hex');
+}
+
+export function domainOf(url) {
+ try {
+ return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+ } catch {
+ return '';
+ }
+}
+
+export function estimateTokens(text) {
+ return Math.ceil(String(text || '').length / CHARS_PER_TOKEN);
+}
+
+const clamp01 = (n) => (Number.isFinite(n) ? Math.min(1, Math.max(0, n)) : null);
+const str = (v, max) => (typeof v === 'string' ? v.replace(/\s+/g, ' ').trim().slice(0, max) : '');
+
+// A visible link rectangle, in normalized viewport coordinates (0..1). The page
+// renders the avatar walking to these, so a malformed rect must be dropped, never
+// passed through to a client that would place a body off-canvas.
+export function sanitizeLinkRect(link) {
+ if (!link || typeof link !== 'object') return null;
+ const x = clamp01(Number(link.x));
+ const y = clamp01(Number(link.y));
+ const w = clamp01(Number(link.w));
+ const h = clamp01(Number(link.h));
+ if (x == null || y == null || w == null || h == null || w <= 0 || h <= 0) return null;
+ const out = { x, y, w: Math.min(w, 1 - x), h: Math.min(h, 1 - y), t: str(link.t, 80) };
+ if (out.w <= 0 || out.h <= 0) return null;
+ return out;
+}
+
+// The live step: what a crawler is doing right now. Lite on purpose (no image
+// bytes): the fleet stream fans it out to every viewer on every change.
+export function sanitizeStep(step) {
+ if (!step || typeof step !== 'object') return null;
+ const url = normalizeUrl(step.url);
+ const links = Array.isArray(step.links)
+ ? step.links.slice(0, MAX_LINKS).map(sanitizeLinkRect).filter(Boolean)
+ : [];
+ const t = Number(step.target);
+ const target = Number.isInteger(t) && t >= 0 && t < links.length ? t : null;
+ const nextUrl = target != null ? normalizeUrl(step.nextUrl) : null;
+ const status = STEP_STATUSES.includes(step.status) ? step.status : 'reading';
+ const seq = Number(step.seq);
+ return {
+ url,
+ domain: url ? domainOf(url) : '',
+ title: str(step.title, TITLE_MAX),
+ thought: str(step.thought, THOUGHT_MAX),
+ status,
+ links,
+ target,
+ nextUrl,
+ seq: Number.isSafeInteger(seq) && seq >= 0 ? seq : 0,
+ scrollY: clamp01(Number(step.scrollY)) ?? 0,
+ };
+}
+
+// A finished page read: the corpus row. Text is bounded here so a runaway
+// page cannot push megabytes through the API.
+export function sanitizePage(page) {
+ if (!page || typeof page !== 'object') return null;
+ const url = normalizeUrl(page.url);
+ if (!url) return null;
+ const text = typeof page.text === 'string' ? page.text.slice(0, MAX_TEXT_CHARS) : '';
+ const rel = Number(page.relevance);
+ const linksOut = Number(page.linksOut);
+ return {
+ url,
+ hash: urlHash(url),
+ domain: domainOf(url),
+ title: str(page.title, TITLE_MAX) || null,
+ gist: str(page.gist, GIST_MAX) || null,
+ text,
+ tokens: estimateTokens(text),
+ linksOut: Number.isSafeInteger(linksOut) && linksOut >= 0 ? Math.min(linksOut, 100_000) : 0,
+ relevance: Number.isFinite(rel) ? Math.min(1, Math.max(0, rel)) : null,
+ fromUrl: normalizeUrl(page.fromUrl),
+ };
+}
+
+export function isFrameB64(s) {
+ return typeof s === 'string' && s.length > 64 && s.length <= FRAME_B64_MAX && /^[A-Za-z0-9+/]+=*$/.test(s)
+ && s.startsWith('/9j/'); // JPEG magic (FF D8 FF) in base64
+}
+
+// Owner-set mission. Seeds must be public http(s) URLs; the worker re-guards every
+// navigation against private hosts, this is the first fence at write time.
+export function validateMission(body) {
+ const topic = str(body?.topic, TOPIC_MAX + 1);
+ if (topic.length < TOPIC_MIN || topic.length > TOPIC_MAX) {
+ return { error: `topic must be ${TOPIC_MIN} to ${TOPIC_MAX} characters` };
+ }
+ const rawSeeds = Array.isArray(body?.seeds)
+ ? body.seeds
+ : typeof body?.seeds === 'string' ? body.seeds.split(/[\s,]+/) : [];
+ const seeds = [];
+ for (const s of rawSeeds) {
+ if (!s || typeof s !== 'string') continue;
+ const withScheme = /^https?:\/\//i.test(s.trim()) ? s.trim() : `https://${s.trim()}`;
+ const n = normalizeUrl(withScheme);
+ if (!n) return { error: `not a web address: ${s.slice(0, 80)}` };
+ const host = new URL(n).hostname;
+ if (isPrivateHostname(host)) return { error: `not a public address: ${host}` };
+ if (!seeds.includes(n)) seeds.push(n);
+ if (seeds.length > MAX_SEEDS) return { error: `at most ${MAX_SEEDS} start pages` };
+ }
+ return { topic, seeds, enabled: body?.enabled !== false };
+}
+
+// Static hostname screen (no DNS). Catches literals and the well-known internal
+// names; the worker additionally resolves every host before it navigates.
+export function isPrivateHostname(host) {
+ const h = String(host || '').toLowerCase().replace(/^\[|\]$/g, '');
+ if (!h || (!h.includes('.') && !h.includes(':'))) return true;
+ if (h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.local') || h.endsWith('.internal')) return true;
+ if (h === 'metadata.google.internal' || h === 'metadata') return true;
+ const v4 = h.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
+ if (v4) {
+ const [a, b] = [Number(v4[1]), Number(v4[2])];
+ return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31)
+ || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224;
+ }
+ if (h.includes(':')) {
+ return h === '::1' || h === '::' || h.startsWith('fc') || h.startsWith('fd') || h.startsWith('fe80') || h.startsWith('::ffff:');
+ }
+ return false;
+}
+
+// The memory an owner's agent keeps from a page it read, so its chat can recall
+// it later ("what have you been reading about?").
+export function pageMemory(page, topic) {
+ const head = page.title ? `Read "${page.title}"` : 'Read a page';
+ const body = page.gist ? `: ${page.gist}` : '.';
+ return `${head} (${page.url}) while crawling for ${topic}${body}`.slice(0, 900);
+}
diff --git a/api/_lib/migrations/20261009230000_agent_crawl.sql b/api/_lib/migrations/20261009230000_agent_crawl.sql
new file mode 100644
index 000000000..8c84b63a2
--- /dev/null
+++ b/api/_lib/migrations/20261009230000_agent_crawl.sql
@@ -0,0 +1,53 @@
+-- Migration: the Crawl, agents that read the open web in real browsers.
+-- Apply: node scripts/apply-migrations.mjs --apply --file 20261009230000_agent_crawl.sql
+-- Idempotent.
+--
+-- crawl_missions is the roster: one row per agent sent out to read, holding the
+-- topic it reads toward and the seed URLs it starts from. Only the agent's owner
+-- writes a row (api/crawl/[action].js action=mission), and the always-on worker
+-- (workers/agent-crawler) reads the enabled rows to decide who walks.
+--
+-- crawl_pages is the corpus: one row per page an agent finished reading, with a
+-- short extractive gist and a token count. The full cleaned text lives in object
+-- storage under text_key so the table stays small enough for Neon; a page with
+-- no storage configured keeps its gist and drops the body.
+
+create table if not exists crawl_missions (
+ agent_id uuid primary key references agent_identities(id) on delete cascade,
+ topic text not null check (char_length(topic) between 2 and 120),
+ seeds text[] not null default '{}',
+ enabled boolean not null default true,
+ set_by uuid references users(id) on delete set null,
+ pages_read bigint not null default 0,
+ tokens_read bigint not null default 0,
+ last_url text,
+ last_at timestamptz,
+ created_at timestamptz not null default now(),
+ updated_at timestamptz not null default now()
+);
+
+create index if not exists crawl_missions_enabled_idx
+ on crawl_missions (last_at asc nulls first)
+ where enabled = true;
+
+create table if not exists crawl_pages (
+ id bigserial primary key,
+ agent_id uuid not null references agent_identities(id) on delete cascade,
+ url text not null,
+ url_hash text not null,
+ domain text not null,
+ title text,
+ gist text,
+ tokens integer not null default 0,
+ links_out integer not null default 0,
+ relevance real,
+ from_url text,
+ text_key text,
+ created_at timestamptz not null default now(),
+ unique (agent_id, url_hash)
+);
+
+create index if not exists crawl_pages_created_idx on crawl_pages (created_at desc);
+create index if not exists crawl_pages_agent_idx on crawl_pages (agent_id, created_at desc);
+create index if not exists crawl_pages_domain_idx on crawl_pages (domain);
+create index if not exists crawl_pages_url_hash_idx on crawl_pages (url_hash);
diff --git a/api/_lib/rate-limit.js b/api/_lib/rate-limit.js
index 51968a879..ad9d4167f 100644
--- a/api/_lib/rate-limit.js
+++ b/api/_lib/rate-limit.js
@@ -851,6 +851,13 @@ export const limits = {
// distinct override gets its own isolated bucket under the rl:api:ip prefix.
apiIp: (ip, opts = {}) =>
getLimiter('api:ip', { limit: 120, window: '1 m', ...opts }).limit(ip),
+ // The Crawl (api/crawl/[action].js). Frames are fetched once per step per tile
+ // by every viewer of /crawl, so that bucket stays in per-instance memory rather
+ // than spending a Redis command per JPEG; it guards against floods, not spend.
+ // Reads are the stats/crawlers/pages/live lane; missions are the owner write.
+ crawlFrameIp: (ip) => getLimiter('crawl:frame:ip', { limit: 900, window: '1 m', local: true }).limit(ip),
+ crawlReadIp: (ip) => getLimiter('crawl:read:ip', { limit: 240, window: '1 m', local: true }).limit(ip),
+ crawlMissionIp: (ip) => getLimiter('crawl:mission:ip', { limit: 30, window: '10 m' }).limit(ip),
// Free, unauthenticated 3D Studio (api/mcp-studio.js) abuse protection. Every
// studio tool routes through a FREE lane (NVIDIA NIM text→3D, HF Spaces
// image→3D) — zero marginal vendor cost — because the studio never names the
diff --git a/api/crawl/[action].js b/api/crawl/[action].js
new file mode 100644
index 000000000..e44e5ce66
--- /dev/null
+++ b/api/crawl/[action].js
@@ -0,0 +1,606 @@
+// The Crawl: three.ws agents reading the open web in real browsers, watched live.
+// Doc: docs/crawl.md. Worker: workers/agent-crawler. Page: /crawl.
+//
+// Routes:
+// GET /api/crawl/stats fleet + corpus totals, top domains (cached 15s)
+// GET /api/crawl/crawlers enrolled crawlers with avatar + live state
+// GET /api/crawl/live SSE: snapshot, then step/sleep/page events
+// GET /api/crawl/frame?agent=<uuid> latest JPEG of that crawler's browser
+// GET /api/crawl/pages?agent=&limit=&before=&format=json|jsonl
+// the corpus, newest first (jsonl = dataset export)
+// GET /api/crawl/mission?agent=<uuid> one agent's mission (public summary)
+// PUT /api/crawl/mission owner sends an agent out { agentId, topic, seeds[], enabled }
+// DELETE /api/crawl/mission?agent=<uuid> owner calls it home
+// GET /api/crawl/mine the signed-in owner's agents + their missions
+// GET /api/crawl/roster worker: who should be walking (bearer CRAWL_WORKER_SECRET)
+// POST /api/crawl/push worker: one step (+ optional finished page) for one crawler
+//
+// Storage: Redis carries the live layer (crawl:live:<id> step JSON, crawl:frame:<id>
+// base64 JPEG, crawl:active sorted set, crawl:feed recent reads); Postgres carries
+// the roster (crawl_missions) and the corpus (crawl_pages); object storage holds
+// each page's full cleaned text under crawl/text/.
+
+import { timingSafeEqual } from 'node:crypto';
+import { cors, error, json, method, rateLimited, readJson, wrap } from '../_lib/http.js';
+import { limits, clientIp } from '../_lib/rate-limit.js';
+import { getRedis } from '../_lib/redis.js';
+import { sql } from '../_lib/db.js';
+import { isUuid } from '../_lib/validate.js';
+import { getSessionUser, extractBearer } from '../_lib/auth.js';
+import { requireCsrf } from '../_lib/csrf.js';
+import { objectStorageConfigured, putObject, publicUrlOrNull, thumbnailUrl } from '../_lib/r2.js';
+import { resolveAvatarUrl } from '../_lib/avatars.js';
+import {
+ CRAWL_LIVE_TTL, CRAWL_ACTIVE_WINDOW_MS, isFrameB64, pageMemory, sanitizePage, sanitizeStep, validateMission,
+} from '../_lib/crawl.js';
+
+export const maxDuration = 300;
+
+const WORKER_SECRET = process.env.CRAWL_WORKER_SECRET || '';
+const LIVE_KEY = (id) => `crawl:live:${id}`;
+const FRAME_KEY = (id) => `crawl:frame:${id}`;
+const ACTIVE_KEY = 'crawl:active';
+const FEED_KEY = 'crawl:feed';
+const STATS_KEY = 'crawl:stats:v1';
+const FEED_CAP = 60;
+const MEMORY_TTL_MS = 30 * 24 * 3600 * 1000;
+const MEMORY_MIN_RELEVANCE = 0.12;
+
+function isWorker(req) {
+ const bearer = extractBearer(req);
+ if (!bearer || WORKER_SECRET.length < 16) return false;
+ const a = Buffer.from(bearer);
+ const b = Buffer.from(WORKER_SECRET);
+ return a.length === b.length && timingSafeEqual(a, b);
+}
+
+// Upstash's REST client parses JSON values on read, so a record can come back as
+// a string or an already-parsed object. Accept both.
+function parseRecord(v) {
+ if (v && typeof v === 'object') return v;
+ if (typeof v !== 'string') return null;
+ try { return JSON.parse(v); } catch { return null; }
+}
+
+function avatarFields(row) {
+ const isPublic = row.avatar_visibility === 'public' || row.avatar_visibility === 'unlisted';
+ let thumb = null;
+ if (isPublic && row.thumbnail_key) {
+ try { thumb = thumbnailUrl(row.thumbnail_key); } catch { thumb = null; }
+ }
+ return { glbRow: isPublic && row.storage_key ? row : null, thumb };
+}
+
+async function resolveGlb(row) {
+ if (!row) return null;
+ try {
+ const { url } = await resolveAvatarUrl({
+ id: row.avatar_id,
+ storage_key: row.storage_key,
+ baked_storage_key: row.baked_storage_key,
+ appearance_hash: row.appearance_hash,
+ appearance: row.appearance,
+ visibility: row.avatar_visibility,
+ });
+ return url;
+ } catch {
+ return null;
+ }
+}
+
+async function awakeSet(r) {
+ if (!r) return new Map();
+ const now = Date.now();
+ try {
+ const raw = await r.zrange(ACTIVE_KEY, now - CRAWL_ACTIVE_WINDOW_MS, now, { byScore: true, withScores: true });
+ const out = new Map();
+ for (let i = 0; i + 1 < (raw || []).length; i += 2) out.set(String(raw[i]), Number(raw[i + 1]));
+ return out;
+ } catch {
+ return new Map();
+ }
+}
+
+// ── worker: roster ──────────────────────────────────────────────────────────
+async function handleRoster(req, res) {
+ if (!isWorker(req)) return error(res, 401, 'unauthorized', 'crawl worker credential required');
+ const url = new URL(req.url, 'http://x');
+ const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 24));
+ const rows = await sql`
+ SELECT m.agent_id, m.topic, m.seeds, m.pages_read, i.name
+ FROM crawl_missions m
+ JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL
+ WHERE m.enabled = true
+ ORDER BY m.last_at ASC NULLS FIRST
+ LIMIT ${limit}
+ `;
+ return json(res, 200, {
+ crawlers: rows.map((r) => ({
+ agentId: r.agent_id, name: r.name, topic: r.topic, seeds: r.seeds || [], pagesRead: Number(r.pages_read) || 0,
+ })),
+ }, { 'cache-control': 'no-store' });
+}
+
+// ── worker: push ────────────────────────────────────────────────────────────
+async function storeText(page) {
+ if (!page.text || !objectStorageConfigured()) return null;
+ const key = `crawl/text/${page.hash.slice(0, 2)}/${page.hash}.txt`;
+ try {
+ await putObject({ key, body: page.text, contentType: 'text/plain; charset=utf-8', metadata: { url: encodeURI(page.url).slice(0, 1024) } });
+ return key;
+ } catch (err) {
+ console.warn('[crawl] text store failed:', err?.message);
+ return null;
+ }
+}
+
+async function recordPage(agentId, mission, page) {
+ const [row] = await sql`
+ INSERT INTO crawl_pages (agent_id, url, url_hash, domain, title, gist, tokens, links_out, relevance, from_url)
+ VALUES (${agentId}, ${page.url}, ${page.hash}, ${page.domain}, ${page.title}, ${page.gist}, ${page.tokens},
+ ${page.linksOut}, ${page.relevance}, ${page.fromUrl})
+ ON CONFLICT (agent_id, url_hash) DO NOTHING
+ RETURNING id
+ `;
+ if (!row) return { inserted: false };
+
+ const textKey = await storeText(page);
+ const [counts] = await sql`
+ UPDATE crawl_missions
+ SET pages_read = pages_read + 1, tokens_read = tokens_read + ${page.tokens},
+ last_url = ${page.url}, last_at = now(), updated_at = now()
+ WHERE agent_id = ${agentId}
+ RETURNING pages_read, tokens_read
+ `;
+ if (textKey) await sql`UPDATE crawl_pages SET text_key = ${textKey} WHERE id = ${row.id}`;
+
+ // The agent keeps what it read, but only when its own owner sent it out: an
+ // agent's memory is the owner's to shape.
+ const ownerSent = mission.set_by && mission.set_by === mission.user_id;
+ if (ownerSent && page.gist && (page.relevance == null || page.relevance >= MEMORY_MIN_RELEVANCE)) {
+ const salience = 0.25 + 0.4 * (page.relevance ?? 0.3);
+ const expires = new Date(Date.now() + MEMORY_TTL_MS).toISOString();
+ await sql`
+ INSERT INTO agent_memories (agent_id, type, content, tags, context, salience, tier, expires_at)
+ VALUES (${agentId}, 'reference', ${pageMemory(page, mission.topic)}, ${['crawl', page.domain]},
+ ${JSON.stringify({ source: 'crawl', url: page.url, page_id: row.id })}::jsonb, ${salience}, 'archival', ${expires})
+ `.catch((err) => console.warn('[crawl] memory write failed:', err?.message));
+ }
+ return { inserted: true, id: row.id, pagesRead: Number(counts?.pages_read) || 0 };
+}
+
+async function handlePush(req, res) {
+ if (!isWorker(req)) return error(res, 401, 'unauthorized', 'crawl worker credential required');
+ let body;
+ try {
+ body = await readJson(req, 1_200_000);
+ } catch {
+ return error(res, 400, 'invalid_body', 'request body must be JSON under 1.2 MB');
+ }
+ const agentId = String(body?.agentId || '');
+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agentId must be a uuid');
+
+ const [mission] = await sql`
+ SELECT m.topic, m.set_by, m.enabled, m.pages_read, i.name, i.user_id
+ FROM crawl_missions m
+ JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL
+ WHERE m.agent_id = ${agentId}
+ LIMIT 1
+ `;
+ if (!mission || !mission.enabled) return error(res, 410, 'not_enrolled', 'this agent is not out crawling');
+
+ const step = sanitizeStep(body.step);
+ if (!step) return error(res, 400, 'invalid_step', 'step is required');
+ const page = body.page ? sanitizePage(body.page) : null;
+ const frame = isFrameB64(body.frame) ? body.frame : null;
+
+ let pagesRead = Number(mission.pages_read) || 0;
+ let recorded = null;
+ if (page) {
+ recorded = await recordPage(agentId, mission, page);
+ if (recorded.inserted) pagesRead = recorded.pagesRead;
+ }
+
+ const r = getRedis();
+ if (!r) return json(res, 200, { ok: true, live: false, pagesRead });
+
+ const now = Date.now();
+ const live = {
+ agentId, name: mission.name, topic: mission.topic, ts: now, pagesRead,
+ frameTs: frame ? now : undefined, ...step,
+ };
+ if (!frame) {
+ // A step without new pixels keeps pointing at the frame already stored.
+ const prev = parseRecord(await r.get(LIVE_KEY(agentId)).catch(() => null));
+ if (prev?.frameTs) live.frameTs = prev.frameTs;
+ }
+ const writes = [
+ r.set(LIVE_KEY(agentId), JSON.stringify(live), { ex: CRAWL_LIVE_TTL }),
+ r.zadd(ACTIVE_KEY, { score: now, member: agentId }),
+ r.zremrangebyscore(ACTIVE_KEY, 0, now - CRAWL_ACTIVE_WINDOW_MS * 2),
+ ];
+ if (frame) writes.push(r.set(FRAME_KEY(agentId), frame, { ex: CRAWL_LIVE_TTL }));
+ if (recorded?.inserted) {
+ const entry = { agentId, name: mission.name, url: page.url, title: page.title, domain: page.domain, tokens: page.tokens, gist: page.gist, ts: now };
+ writes.push(r.lpush(FEED_KEY, JSON.stringify(entry)).then(() => r.ltrim(FEED_KEY, 0, FEED_CAP - 1)));
+ }
+ await Promise.all(writes);
+ return json(res, 200, { ok: true, live: true, pagesRead, recorded: Boolean(recorded?.inserted) });
+}
+
+// ── public: frame ───────────────────────────────────────────────────────────
+async function handleFrame(req, res) {
+ const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';
+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');
+ const r = getRedis();
+ if (!r) return error(res, 503, 'live_unavailable', 'the live layer is offline');
+ const b64 = await r.get(FRAME_KEY(agentId));
+ if (typeof b64 !== 'string' || !b64) {
+ return error(res, 404, 'asleep', 'this crawler has no live frame right now');
+ }
+ const buf = Buffer.from(b64, 'base64');
+ res.writeHead(200, {
+ 'content-type': 'image/jpeg',
+ 'content-length': buf.length,
+ // The page asks with ?t=<frameTs>, so each frame has its own URL.
+ 'cache-control': 'public, max-age=30',
+ 'x-content-type-options': 'nosniff',
+ });
+ res.end(buf);
+}
+
+// ── public: live SSE ────────────────────────────────────────────────────────
+// One poller per server instance, fanned out to every connected viewer, so Redis
+// load is flat in the number of viewers.
+const hub = { subs: new Set(), timer: null, crawlers: new Map(), feedTs: 0, feed: [] };
+const HUB_POLL_MS = 800;
+
+async function readFleet(r) {
+ const awake = await awakeSet(r);
+ const ids = [...awake.keys()].slice(0, 120);
+ if (!ids.length) return [];
+ const raw = await r.mget(...ids.map(LIVE_KEY));
+ return (raw || []).map(parseRecord).filter(Boolean);
+}
+
+async function readFeed(r, n) {
+ const raw = await r.lrange(FEED_KEY, 0, n - 1).catch(() => []);
+ return (raw || []).map(parseRecord).filter(Boolean);
+}
+
+function broadcast(event, data) {
+ const line = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+ for (const s of hub.subs) s(line);
+}
+
+async function hubTick() {
+ const r = getRedis();
+ if (!r || !hub.subs.size) return;
+ try {
+ const fleet = await readFleet(r);
+ const seen = new Set();
+ for (const c of fleet) {
+ seen.add(c.agentId);
+ const prev = hub.crawlers.get(c.agentId);
+ if (!prev || prev.ts !== c.ts) {
+ hub.crawlers.set(c.agentId, c);
+ broadcast('step', c);
+ }
+ }
+ for (const id of [...hub.crawlers.keys()]) {
+ if (!seen.has(id)) {
+ hub.crawlers.delete(id);
+ broadcast('sleep', { agentId: id });
+ }
+ }
+ const recent = await readFeed(r, 10);
+ const fresh = recent.filter((e) => e.ts > hub.feedTs).reverse();
+ for (const e of fresh) broadcast('page', e);
+ if (recent.length) hub.feedTs = Math.max(hub.feedTs, recent[0].ts);
+ } catch (err) {
+ console.warn('[crawl] hub tick failed:', err?.message);
+ }
+}
+
+function hubStart() {
+ if (hub.timer) return;
+ hub.timer = setInterval(hubTick, HUB_POLL_MS);
+ hub.timer.unref?.();
+}
+
+function hubStop() {
+ if (hub.subs.size || !hub.timer) return;
+ clearInterval(hub.timer);
+ hub.timer = null;
+ hub.crawlers.clear();
+}
+
+async function handleLive(req, res) {
+ const r = getRedis();
+ res.writeHead(200, {
+ 'content-type': 'text/event-stream; charset=utf-8',
+ 'cache-control': 'no-cache, no-transform',
+ connection: 'keep-alive',
+ 'x-accel-buffering': 'no',
+ });
+ res.flushHeaders?.();
+ let open = true;
+ const write = (line) => {
+ if (!open || res.writableEnded) return;
+ try { res.write(line); } catch { open = false; }
+ };
+
+ if (!r) {
+ write(`event: snapshot\ndata: ${JSON.stringify({ crawlers: [], feed: [], live: false })}\n\n`);
+ } else {
+ const [fleet, feed] = await Promise.all([readFleet(r).catch(() => []), readFeed(r, 30)]);
+ for (const c of fleet) hub.crawlers.set(c.agentId, c);
+ if (feed.length) hub.feedTs = Math.max(hub.feedTs, feed[0].ts);
+ write(`event: snapshot\ndata: ${JSON.stringify({ crawlers: fleet, feed, live: true })}\n\n`);
+ }
+
+ hub.subs.add(write);
+ hubStart();
+ const ping = setInterval(() => write('event: ping\ndata: {}\n\n'), 15_000);
+ const end = setTimeout(() => { try { res.end(); } catch { /* closed */ } }, 280_000);
+ const close = () => {
+ open = false;
+ clearInterval(ping);
+ clearTimeout(end);
+ hub.subs.delete(write);
+ hubStop();
+ };
+ req.on('close', close);
+ res.on('close', close);
+}
+
+// ── public: stats ───────────────────────────────────────────────────────────
+async function handleStats(req, res) {
+ const r = getRedis();
+ let cached = null;
+ if (r) cached = parseRecord(await r.get(STATS_KEY).catch(() => null));
+ if (!cached) {
+ const [[totals], [enrolled], top] = await Promise.all([
+ sql`
+ SELECT count(*)::bigint AS pages,
+ coalesce(sum(tokens), 0)::bigint AS tokens,
+ count(DISTINCT domain)::int AS domains,
+ count(DISTINCT url_hash)::bigint AS unique_pages,
+ count(*) FILTER (WHERE created_at > now() - interval '24 hours')::bigint AS pages_24h
+ FROM crawl_pages
+ `,
+ sql`SELECT count(*)::int AS n FROM crawl_missions WHERE enabled = true`,
+ sql`
+ SELECT domain, count(*)::int AS pages
+ FROM crawl_pages
+ WHERE created_at > now() - interval '7 days'
+ GROUP BY domain ORDER BY pages DESC LIMIT 12
+ `,
+ ]);
+ cached = {
+ pagesRead: Number(totals.pages) || 0,
+ uniquePages: Number(totals.unique_pages) || 0,
+ tokens: Number(totals.tokens) || 0,
+ domains: totals.domains || 0,
+ pages24h: Number(totals.pages_24h) || 0,
+ crawlersEnrolled: enrolled.n || 0,
+ topDomains: top.map((t) => ({ domain: t.domain, pages: t.pages })),
+ };
+ if (r) await r.set(STATS_KEY, JSON.stringify(cached), { ex: 15 }).catch(() => {});
+ }
+ const awake = await awakeSet(r);
+ return json(res, 200, { ...cached, crawlersAwake: awake.size, ts: Date.now() }, { 'cache-control': 'public, max-age=10' });
+}
+
+// ── public: crawlers ────────────────────────────────────────────────────────
+async function handleCrawlers(req, res) {
+ const r = getRedis();
+ const [rows, awake] = await Promise.all([
+ sql`
+ SELECT m.agent_id, m.topic, m.pages_read, m.tokens_read, m.last_url, m.last_at, m.enabled,
+ i.name, i.description,
+ a.id AS avatar_id, a.storage_key, a.baked_storage_key, a.appearance_hash, a.appearance,
+ a.thumbnail_key, a.visibility AS avatar_visibility
+ FROM crawl_missions m
+ JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL
+ LEFT JOIN avatars a ON a.id = i.avatar_id AND a.deleted_at IS NULL
+ WHERE m.enabled = true
+ ORDER BY m.last_at DESC NULLS LAST
+ LIMIT 120
+ `,
+ awakeSet(r),
+ ]);
+ const crawlers = await Promise.all(rows.map(async (row) => {
+ const { glbRow, thumb } = avatarFields(row);
+ return {
+ agentId: row.agent_id,
+ name: row.name,
+ description: row.description ? String(row.description).slice(0, 200) : null,
+ topic: row.topic,
+ pagesRead: Number(row.pages_read) || 0,
+ tokensRead: Number(row.tokens_read) || 0,
+ lastUrl: row.last_url,
+ lastAt: row.last_at,
+ awake: awake.has(row.agent_id),
+ avatarUrl: await resolveGlb(glbRow),
+ thumbnail: thumb,
+ };
+ }));
+ crawlers.sort((a, b) => Number(b.awake) - Number(a.awake) || b.pagesRead - a.pagesRead);
+ return json(res, 200, { crawlers }, { 'cache-control': 'public, max-age=10' });
+}
+
+// ── public: pages (corpus) ──────────────────────────────────────────────────
+async function handlePages(req, res) {
+ const p = new URL(req.url, 'http://x').searchParams;
+ const agent = p.get('agent');
+ if (agent && !isUuid(agent)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');
+ const limit = Math.min(500, Math.max(1, Number(p.get('limit')) || 40));
+ const before = Number(p.get('before'));
+ const beforeId = Number.isSafeInteger(before) && before > 0 ? before : null;
+ const rows = await sql`
+ SELECT p.id, p.agent_id, i.name AS agent_name, p.url, p.domain, p.title, p.gist, p.tokens,
+ p.links_out, p.relevance, p.from_url, p.text_key, p.created_at
+ FROM crawl_pages p
+ JOIN agent_identities i ON i.id = p.agent_id
+ WHERE (${agent}::uuid IS NULL OR p.agent_id = ${agent}::uuid)
+ AND (${beforeId}::bigint IS NULL OR p.id < ${beforeId}::bigint)
+ ORDER BY p.id DESC
+ LIMIT ${limit}
+ `;
+ const pages = rows.map((row) => ({
+ id: Number(row.id),
+ agentId: row.agent_id,
+ agentName: row.agent_name,
+ url: row.url,
+ domain: row.domain,
+ title: row.title,
+ gist: row.gist,
+ tokens: row.tokens,
+ linksOut: row.links_out,
+ relevance: row.relevance,
+ fromUrl: row.from_url,
+ textUrl: publicUrlOrNull(row.text_key),
+ readAt: row.created_at,
+ }));
+ if (p.get('format') === 'jsonl') {
+ res.writeHead(200, {
+ 'content-type': 'application/x-ndjson; charset=utf-8',
+ 'cache-control': 'public, max-age=30',
+ 'content-disposition': 'inline; filename="three-ws-crawl.jsonl"',
+ });
+ res.end(pages.map((x) => JSON.stringify(x)).join('\n') + (pages.length ? '\n' : ''));
+ return;
+ }
+ const next = pages.length === limit ? pages[pages.length - 1].id : null;
+ return json(res, 200, { pages, next }, { 'cache-control': 'public, max-age=10' });
+}
+
+// ── mission (public read, owner write) ──────────────────────────────────────
+async function ownedAgent(agentId, userId) {
+ const [row] = await sql`
+ SELECT id, name FROM agent_identities
+ WHERE id = ${agentId} AND user_id = ${userId} AND deleted_at IS NULL
+ LIMIT 1
+ `;
+ return row || null;
+}
+
+function missionView(row) {
+ return {
+ agentId: row.agent_id,
+ name: row.name,
+ topic: row.topic,
+ seeds: row.seeds || [],
+ enabled: row.enabled,
+ pagesRead: Number(row.pages_read) || 0,
+ tokensRead: Number(row.tokens_read) || 0,
+ lastUrl: row.last_url,
+ lastAt: row.last_at,
+ };
+}
+
+async function handleMission(req, res) {
+ if (req.method === 'GET') {
+ const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';
+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');
+ const [row] = await sql`
+ SELECT m.*, i.name FROM crawl_missions m
+ JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL
+ WHERE m.agent_id = ${agentId} LIMIT 1
+ `;
+ if (!row) return json(res, 200, { mission: null }, { 'cache-control': 'no-store' });
+ return json(res, 200, { mission: missionView(row) }, { 'cache-control': 'no-store' });
+ }
+
+ const auth = await getSessionUser(req, res);
+ const userId = auth?.id;
+ if (!userId) return error(res, 401, 'unauthorized', 'sign in to send your agent out');
+ if (!(await requireCsrf(req, res, userId))) return;
+ const rl = await limits.crawlMissionIp(clientIp(req));
+ if (!rl.success) return rateLimited(res, rl);
+
+ if (req.method === 'DELETE') {
+ const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';
+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');
+ if (!(await ownedAgent(agentId, userId))) return error(res, 403, 'forbidden', 'you do not own this agent');
+ await sql`UPDATE crawl_missions SET enabled = false, updated_at = now() WHERE agent_id = ${agentId}`;
+ const r = getRedis();
+ if (r) await Promise.all([r.del(LIVE_KEY(agentId)), r.zrem(ACTIVE_KEY, agentId)]).catch(() => {});
+ return json(res, 200, { ok: true, enabled: false });
+ }
+
+ let body;
+ try { body = await readJson(req, 16_000); } catch { return error(res, 400, 'invalid_body', 'body must be JSON'); }
+ const agentId = String(body?.agentId || '');
+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agentId must be a uuid');
+ const agent = await ownedAgent(agentId, userId);
+ if (!agent) return error(res, 403, 'forbidden', 'you do not own this agent');
+ const m = validateMission(body);
+ if (m.error) return error(res, 400, 'invalid_mission', m.error);
+
+ const [row] = await sql`
+ INSERT INTO crawl_missions (agent_id, topic, seeds, enabled, set_by)
+ VALUES (${agentId}, ${m.topic}, ${m.seeds}, ${m.enabled}, ${userId})
+ ON CONFLICT (agent_id) DO UPDATE
+ SET topic = excluded.topic, seeds = excluded.seeds, enabled = excluded.enabled,
+ set_by = excluded.set_by, updated_at = now()
+ RETURNING *
+ `;
+ return json(res, 200, { mission: missionView({ ...row, name: agent.name }) });
+}
+
+async function handleMine(req, res) {
+ const auth = await getSessionUser(req, res);
+ const userId = auth?.id;
+ if (!userId) return error(res, 401, 'unauthorized', 'sign in to see your agents');
+ const rows = await sql`
+ SELECT i.id, i.name, a.thumbnail_key, a.visibility AS avatar_visibility,
+ m.topic, m.seeds, m.enabled, m.pages_read, m.tokens_read, m.last_url, m.last_at
+ FROM agent_identities i
+ LEFT JOIN avatars a ON a.id = i.avatar_id AND a.deleted_at IS NULL
+ LEFT JOIN crawl_missions m ON m.agent_id = i.id
+ WHERE i.user_id = ${userId} AND i.deleted_at IS NULL
+ ORDER BY m.enabled DESC NULLS LAST, i.created_at DESC
+ LIMIT 60
+ `;
+ return json(res, 200, {
+ agents: rows.map((row) => ({
+ agentId: row.id,
+ name: row.name,
+ thumbnail: avatarFields(row).thumb,
+ mission: row.topic ? missionView({ ...row, agent_id: row.id }) : null,
+ })),
+ }, { 'cache-control': 'no-store' });
+}
+
+const ROUTES = {
+ roster: { methods: ['GET'], fn: handleRoster },
+ push: { methods: ['POST'], fn: handlePush },
+ frame: { methods: ['GET'], fn: handleFrame },
+ live: { methods: ['GET'], fn: handleLive },
+ stats: { methods: ['GET'], fn: handleStats },
+ crawlers: { methods: ['GET'], fn: handleCrawlers },
+ pages: { methods: ['GET'], fn: handlePages },
+ mission: { methods: ['GET', 'PUT', 'DELETE'], fn: handleMission },
+ mine: { methods: ['GET'], fn: handleMine },
+};
+
+export default wrap(async function handleCrawl(req, res) {
+ const action = new URL(req.url, 'http://x').searchParams.get('action') || '';
+ const route = ROUTES[action];
+ if (cors(req, res, { methods: `${route ? route.methods.join(',') : 'GET'},OPTIONS` })) return;
+ if (!route) return error(res, 404, 'not_found', 'unknown crawl action');
+ if (!method(req, res, route.methods)) return;
+
+ // Worker traffic is authenticated by secret and paced by the worker itself.
+ // Frames get a wider bucket (a wall of tiles fetches one per step each).
+ if (action === 'frame') {
+ const rl = await limits.crawlFrameIp(clientIp(req));
+ if (!rl.success) return rateLimited(res, rl);
+ } else if (action !== 'push' && action !== 'roster') {
+ const rl = await limits.crawlReadIp(clientIp(req));
+ if (!rl.success) return rateLimited(res, rl);
+ }
+ return route.fn(req, res);
+});
diff --git a/vercel.json b/vercel.json
index 0866ad086..14530e47c 100644
--- a/vercel.json
+++ b/vercel.json
@@ -1662,6 +1662,10 @@
"src": "/rig-doctor/?",
"dest": "/rig-doctor.html"
},
+ {
+ "src": "/crawl/?",
+ "dest": "/crawl.html"
+ },
{
"src": "/avatar-artifact/?",
"dest": "/avatar-artifact.html"
@@ -3932,6 +3936,42 @@
"src": "/api/agent-screen-active",
"dest": "/api/agent-screen-active.js"
},
+ {
+ "src": "/api/crawl/stats",
+ "dest": "/api/crawl/[action]?action=stats"
+ },
+ {
+ "src": "/api/crawl/crawlers",
+ "dest": "/api/crawl/[action]?action=crawlers"
+ },
+ {
+ "src": "/api/crawl/live",
+ "dest": "/api/crawl/[action]?action=live"
+ },
+ {
+ "src": "/api/crawl/frame",
+ "dest": "/api/crawl/[action]?action=frame"
+ },
+ {
+ "src": "/api/crawl/pages",
+ "dest": "/api/crawl/[action]?action=pages"
+ },
+ {
+ "src": "/api/crawl/mission",
+ "dest": "/api/crawl/[action]?action=mission"
+ },
+ {
+ "src": "/api/crawl/mine",
+ "dest": "/api/crawl/[action]?action=mine"
+ },
+ {
+ "src": "/api/crawl/roster",
+ "dest": "/api/crawl/[action]?action=roster"
+ },
+ {
+ "src": "/api/crawl/push",
+ "dest": "/api/crawl/[action]?action=push"
+ },
{
"src": "/api/agent-task",
"dest": "/api/agent-task.js"
diff --git a/workers/agent-crawler/.dockerignore b/workers/agent-crawler/.dockerignore
new file mode 100644
index 000000000..158d407b4
--- /dev/null
+++ b/workers/agent-crawler/.dockerignore
@@ -0,0 +1,6 @@
+node_modules
+npm-debug.log
+README.md
+Dockerfile
+.dockerignore
+cloudbuild.yaml
diff --git a/workers/agent-crawler/Dockerfile b/workers/agent-crawler/Dockerfile
new file mode 100644
index 000000000..a1ae638f5
--- /dev/null
+++ b/workers/agent-crawler/Dockerfile
@@ -0,0 +1,27 @@
+# syntax=docker/dockerfile:1
+# agent-crawler: three.ws agents reading the open web in real browsers (/crawl).
+#
+# The base image ships Chromium plus every system dependency it needs. Its tag
+# MUST match the playwright version pinned (exactly) in package.json and the
+# lockfile: the image bakes the browser build that playwright release expects,
+# so a drifted pair dies on launch with "Executable doesn't exist at
+# /ms-playwright/chromium-<rev>". Bump both together.
+FROM mcr.microsoft.com/playwright:v1.61.1-noble
+
+WORKDIR /app
+
+COPY package.json package-lock.json ./
+RUN npm ci --omit=dev
+
+COPY index.js crawler.js frontier.js extract.js net.js seeds.js ./
+
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+ENV NODE_ENV=production \
+ BASE_URL=https://three.ws \
+ CRAWLERS=8 \
+ JPEG_QUALITY=55
+
+# CRAWL_WORKER_SECRET must be supplied at runtime.
+USER pwuser
+
+CMD ["node", "index.js"]
diff --git a/workers/agent-crawler/cloudbuild.yaml b/workers/agent-crawler/cloudbuild.yaml
new file mode 100644
index 000000000..128b4f21f
--- /dev/null
+++ b/workers/agent-crawler/cloudbuild.yaml
@@ -0,0 +1,80 @@
+# Cloud Build to Cloud Run for the crawler fleet behind /crawl (workers/agent-crawler).
+#
+# Background daemon deployed as a SERVICE: the roster loop runs continuously and
+# index.js binds a liveness endpoint on $PORT so the Cloud Run startup probe
+# passes. --no-cpu-throttling and --min-instances=1 keep the crawlers walking
+# between probes; --max-instances=1 because two instances would both claim the
+# same roster and push interleaved steps for the same agent.
+#
+# Build context is THIS directory:
+#
+# gcloud builds submit workers/agent-crawler \
+# --config workers/agent-crawler/cloudbuild.yaml \
+# --region us-central1 --project aerial-vehicle-466722-p5 \
+# --substitutions=SHORT_SHA=manual$(date +%s)
+#
+# The shared secret lives in Secret Manager as `crawl-worker-secret` (three-ws@
+# holds secretAccessor). The API reads the same value as CRAWL_WORKER_SECRET;
+# both must match or every roster poll answers 401.
+#
+# Builds run as the pinned three-ws-build service account: the project's default
+# compute service account was deleted.
+serviceAccount: projects/aerial-vehicle-466722-p5/serviceAccounts/[email protected]
+
+substitutions:
+ _REGION: us-central1
+ _SERVICE: agent-crawler
+ _REPO: workers
+ _BASE_URL: https://three.ws
+ # Agents walking at once. Each holds one browser context (~300 MB peak with a
+ # heavy page), so 8 fits 4Gi with headroom. Raise memory with it.
+ _CRAWLERS: "8"
+
+steps:
+ - id: build
+ name: gcr.io/cloud-builders/docker
+ args:
+ - build
+ - --tag
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:$SHORT_SHA
+ - --tag
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:latest
+ - --cache-from
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:latest
+ - .
+ timeout: 1200s
+
+ - id: push
+ name: gcr.io/cloud-builders/docker
+ args:
+ - push
+ - --all-tags
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}
+
+ - id: deploy
+ name: gcr.io/google.com/cloudsdktool/cloud-sdk:slim
+ entrypoint: gcloud
+ args:
+ - run
+ - deploy
+ - ${_SERVICE}
+ - --image=${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:$SHORT_SHA
+ - --region=${_REGION}
+ - --platform=managed
+ - --no-cpu-throttling
+ - --cpu=2
+ - --memory=4Gi
+ - --min-instances=1
+ - --max-instances=1
+ - --no-allow-unauthenticated
+ - --service-account=three-ws@$PROJECT_ID.iam.gserviceaccount.com
+ - --set-env-vars=NODE_ENV=production,BASE_URL=${_BASE_URL},CRAWLERS=${_CRAWLERS}
+ - --set-secrets=CRAWL_WORKER_SECRET=crawl-worker-secret:latest
+
+images:
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:$SHORT_SHA
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:latest
+
+options:
+ machineType: E2_HIGHCPU_8
+ logging: CLOUD_LOGGING_ONLY
diff --git a/workers/agent-crawler/crawler.js b/workers/agent-crawler/crawler.js
new file mode 100644
index 000000000..9e8990ac9
--- /dev/null
+++ b/workers/agent-crawler/crawler.js
@@ -0,0 +1,375 @@
+// One agent out reading: a browser context of its own, a frontier of leads, and
+// a loop that reads a page, chooses a link, walks to it on screen, and leaps.
+//
+// Every phase is pushed to the API as a live step (api/crawl/[action].js
+// action=push) so /crawl can draw the agent's avatar walking across the real
+// screenshot to the link it really chose. The page it finished reading rides
+// along with the "reading" step and lands in the open corpus.
+
+import { readPage, revealLink, visibleLinks, CONSENT_CSS } from './extract.js';
+import {
+ Frontier, canonical, gistOf, hostOf, relevance, scoreLink, thoughtFor, topicTerms,
+} from './frontier.js';
+import { USER_AGENT, claimDomain, domainReadyIn, guardHost, robotsCheck } from './net.js';
+import { searchSeeds } from './seeds.js';
+
+export const VIEWPORT = { width: 1200, height: 750 };
+const STEP_LINKS = 48;
+const MIN_TEXT_FOR_CORPUS = 200;
+const FRAME_B64_MAX = 690_000;
+const RECYCLE_PAGE_EVERY = 40;
+
+export class NotEnrolledError extends Error {}
+
+const sleep = (ms) => new Promise((r) => { setTimeout(r, ms); });
+
+export class Crawler {
+ constructor({ mission, browser, api, log, opts }) {
+ this.agentId = mission.agentId;
+ this.name = mission.name;
+ this.topic = mission.topic;
+ this.seeds = mission.seeds || [];
+ this.terms = topicTerms(mission.topic);
+ this.browser = browser;
+ this.api = api;
+ this.opts = opts;
+ this.log = (msg) => log(`[${this.name}] ${msg}`);
+ this.frontier = new Frontier();
+ this.visited = new Set();
+ this.domainCounts = new Map();
+ this.seq = 0;
+ this.stopped = false;
+ this.startedAt = Date.now();
+ this.pagesRead = mission.pagesRead || 0;
+ this.navigations = 0;
+ this.currentUrl = null;
+ this.status = 'starting';
+ this.lastReseedAt = 0;
+ this.done = null;
+ }
+
+ summary() {
+ return {
+ agentId: this.agentId, name: this.name, topic: this.topic, status: this.status,
+ url: this.currentUrl, frontier: this.frontier.size, visited: this.visited.size,
+ pagesRead: this.pagesRead, upMs: Date.now() - this.startedAt,
+ };
+ }
+
+ updateMission(mission) {
+ if (mission.topic !== this.topic) {
+ this.topic = mission.topic;
+ this.terms = topicTerms(mission.topic);
+ this.frontier = new Frontier();
+ this.log(`topic changed to "${mission.topic}", starting fresh`);
+ }
+ this.seeds = mission.seeds || [];
+ }
+
+ start() {
+ this.done = this.run().catch((err) => {
+ if (!(err instanceof NotEnrolledError)) this.log(`stopped on error: ${err?.message || err}`);
+ }).finally(() => this.close());
+ return this.done;
+ }
+
+ async stop() {
+ this.stopped = true;
+ await this.close();
+ await this.done?.catch(() => {});
+ }
+
+ async close() {
+ const ctx = this.context;
+ this.context = null;
+ this.page = null;
+ if (ctx) await ctx.close().catch(() => {});
+ }
+
+ async openContext() {
+ this.context = await this.browser.newContext({
+ viewport: VIEWPORT,
+ deviceScaleFactor: 1,
+ userAgent: USER_AGENT,
+ locale: 'en-US',
+ acceptDownloads: false,
+ javaScriptEnabled: true,
+ serviceWorkers: 'block',
+ });
+ await this.context.addInitScript((css) => {
+ const add = () => {
+ if (document.getElementById('tws-crawl-consent')) return;
+ const s = document.createElement('style');
+ s.id = 'tws-crawl-consent';
+ s.textContent = css;
+ (document.head || document.documentElement).appendChild(s);
+ };
+ if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', add, { once: true });
+ else add();
+ }, CONSENT_CSS);
+ // Every request is fenced: private hosts are refused and media streams
+ // (which never show in a still frame) are skipped.
+ await this.context.route('**/*', async (route) => {
+ const req = route.request();
+ if (req.resourceType() === 'media') return route.abort('blockedbyclient');
+ let host = '';
+ try { host = new URL(req.url()).hostname; } catch { return route.abort('blockedbyclient'); }
+ if (req.url().startsWith('data:') || req.url().startsWith('blob:')) return route.continue();
+ if (!(await guardHost(host))) return route.abort('blockedbyclient');
+ return route.continue();
+ });
+ this.context.on('page', (p) => {
+ if (this.page && p !== this.page) p.close().catch(() => {});
+ });
+ await this.newPage();
+ }
+
+ async newPage() {
+ if (this.page) await this.page.close().catch(() => {});
+ this.page = await this.context.newPage();
+ this.page.on('dialog', (d) => d.dismiss().catch(() => {}));
+ this.navigations = 0;
+ }
+
+ async run() {
+ await this.openContext();
+ await this.loadHistory();
+ await this.seed();
+
+ // A walked-to link is visited straight away (the avatar is standing on
+ // it); otherwise the frontier picks the next lead.
+ let following = null;
+ while (!this.stopped) {
+ const next = following || this.pickNext();
+ following = null;
+ if (!next) {
+ await this.reseedOrRest();
+ continue;
+ }
+ following = await this.visit(next);
+ }
+ }
+
+ // Pages this agent already read survive restarts: the corpus is the memory.
+ async loadHistory() {
+ try {
+ const pages = await this.api.recentPages(this.agentId, 500);
+ for (const p of pages) {
+ const key = canonical(p.url);
+ if (key) this.visited.add(key);
+ const h = hostOf(p.url);
+ this.domainCounts.set(h, Math.min(10, (this.domainCounts.get(h) || 0) + 1));
+ }
+ } catch (err) {
+ this.log(`history unavailable (${err?.message || err}); starting with a clean slate`);
+ }
+ }
+
+ async seed() {
+ for (const url of this.seeds) this.frontier.add(url, 50, null, 'start page');
+ if (!this.frontier.size) await this.searchForLeads();
+ }
+
+ async searchForLeads() {
+ this.lastReseedAt = Date.now();
+ const { urls, source } = await searchSeeds(this.topic, this.log);
+ urls.forEach((u, i) => this.frontier.add(u, 20 - i * 0.5, null, ''));
+ if (urls.length) this.log(`${urls.length} leads from ${source}`);
+ return urls.length;
+ }
+
+ async reseedOrRest() {
+ const sinceReseed = Date.now() - this.lastReseedAt;
+ if (sinceReseed > 120_000) {
+ for (const url of this.seeds) if (!this.visited.has(canonical(url))) this.frontier.add(url, 50);
+ if (this.frontier.size || (await this.searchForLeads())) return;
+ }
+ this.status = 'resting';
+ await this.pushStep({
+ url: this.currentUrl, title: '', status: 'resting', links: [], target: null,
+ thought: thoughtFor('resting', { reason: `Out of fresh leads on ${this.topic}. Searching again shortly.` }),
+ });
+ await this.idle(60_000);
+ }
+
+ async idle(ms) {
+ const until = Date.now() + ms;
+ while (!this.stopped && Date.now() < until) await sleep(Math.min(1000, until - Date.now()));
+ }
+
+ pickNext() {
+ const accept = (item) => {
+ if (this.visited.has(item.url)) return false;
+ return domainReadyIn(hostOf(item.url)) <= 2500;
+ };
+ let item = this.frontier.pop(accept);
+ if (!item && this.frontier.size) {
+ // Everything left is on a cooling domain: take the best one; visit()
+ // waits out its politeness gap before navigating.
+ item = this.frontier.pop((i) => !this.visited.has(i.url));
+ }
+ return item;
+ }
+
+ async pushStep(step, extra = {}) {
+ this.seq += 1;
+ return this.api.push({ agentId: this.agentId, step: { ...step, seq: this.seq }, ...extra });
+ }
+
+ async screenshot() {
+ let quality = this.opts.jpegQuality;
+ for (let attempt = 0; attempt < 3; attempt++) {
+ const buf = await this.page.screenshot({ type: 'jpeg', quality, timeout: 8000, animations: 'disabled', caret: 'hide' });
+ const b64 = buf.toString('base64');
+ if (b64.length <= FRAME_B64_MAX) return b64;
+ quality = Math.max(25, quality - 15);
+ }
+ return null;
+ }
+
+ async blocked(host, reason) {
+ this.status = 'blocked';
+ await this.pushStep({
+ url: this.currentUrl, title: '', status: 'blocked', links: [], target: null,
+ thought: thoughtFor('blocked', { host, reason }),
+ }).catch((err) => { if (err instanceof NotEnrolledError) throw err; });
+ }
+
+ // Read one page, then decide where to go. Returns the frontier entry to visit
+ // next when the agent chose an on-screen link (it already walked there), or
+ // null to let the frontier decide.
+ async visit(target) {
+ const url = canonical(target.url);
+ if (!url) return null;
+ const host = hostOf(url);
+ this.visited.add(url);
+
+ const robots = await robotsCheck(url).catch(() => ({ allowed: false, delayMs: 0 }));
+ if (!robots.allowed) {
+ await this.blocked(host, 'asks crawlers to stay out of that page');
+ return null;
+ }
+ const wait = domainReadyIn(host);
+ if (wait > 0) await sleep(wait);
+ claimDomain(host, Math.max(this.opts.domainGapMs, robots.delayMs));
+
+ if (this.navigations >= RECYCLE_PAGE_EVERY) await this.newPage();
+ this.navigations += 1;
+
+ let resp;
+ try {
+ resp = await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.opts.navTimeoutMs });
+ } catch (err) {
+ const msg = String(err?.message || err);
+ await this.blocked(host, /ERR_BLOCKED_BY_CLIENT/.test(msg) ? 'is not a public address' : 'did not answer in time');
+ if (/Target closed|crashed/i.test(msg)) await this.newPage();
+ return null;
+ }
+ if (!resp || resp.status() >= 400) {
+ await this.blocked(host, `answered ${resp ? resp.status() : 'nothing'}`);
+ return null;
+ }
+ const ctype = resp.headers()['content-type'] || '';
+ if (ctype && !/html|xhtml/i.test(ctype)) {
+ await this.blocked(host, 'sent a file, not a page');
+ return null;
+ }
+ // Let the page paint its above-the-fold content before the frame is taken.
+ await this.page.waitForLoadState('load', { timeout: 6000 }).catch(() => {});
+
+ const finalUrl = canonical(this.page.url()) || url;
+ this.visited.add(finalUrl);
+ const finalHost = hostOf(finalUrl);
+ this.domainCounts.set(finalHost, (this.domainCounts.get(finalHost) || 0) + 1);
+ this.currentUrl = finalUrl;
+
+ const data = await this.page.evaluate(readPage, 600);
+ const rel = relevance(this.terms, data.title, data.text);
+ const gist = gistOf(data.text, this.terms);
+
+ // Score every link on the page into the frontier. Links found on a
+ // relevant page inherit some of its relevance (topical locality).
+ const ctx = { terms: this.terms, fromHost: finalHost, domainCounts: this.domainCounts, visited: this.visited, rand: Math.random };
+ let best = null;
+ for (const link of data.links) {
+ const s = scoreLink(link, ctx);
+ if (!Number.isFinite(s)) continue;
+ const score = s + rel * 1.5;
+ this.frontier.add(link.href, score, finalUrl, link.text);
+ if (!best || score > best.score) best = { ...link, score };
+ }
+ this.frontier.decay();
+
+ const frame = await this.screenshot();
+ const shown = data.links.filter((l) => l.visible).slice(0, STEP_LINKS).map(({ x, y, w, h, text }) => ({ x, y, w, h, t: text }));
+ const page = data.text.length >= MIN_TEXT_FOR_CORPUS ? {
+ url: finalUrl, title: data.title, text: data.text, gist, linksOut: data.links.length, relevance: rel, fromUrl: target.from,
+ } : null;
+
+ this.status = 'reading';
+ const res = await this.pushStep({
+ url: finalUrl, title: data.title, status: 'reading', links: shown, target: null, scrollY: data.scrollY,
+ thought: thoughtFor('reading', { title: data.title || finalHost, domain: finalHost, topic: this.topic, relevance: rel }),
+ }, { frame, page });
+ if (res?.recorded) this.pagesRead = res.pagesRead || this.pagesRead + 1;
+
+ // Read: dwell on the page roughly in proportion to how much there is.
+ await this.idle(this.opts.readMs + Math.min(3000, data.text.length / 20));
+ if (this.stopped) return null;
+
+ const frontierBest = [...this.frontier.items.values()]
+ .filter((i) => !this.visited.has(i.url) && i.url !== canonical(best?.href))
+ .reduce((m, i) => (!m || i.score > m.score ? i : m), null);
+ if (best && (!frontierBest || best.score >= frontierBest.score - 0.5)) {
+ const walked = await this.walkTo(best, finalUrl, finalHost);
+ if (walked) return walked;
+ }
+
+ // Leap: the best lead is not on this screen.
+ const lead = frontierBest;
+ if (lead) {
+ this.status = 'leaping';
+ await this.pushStep({
+ url: finalUrl, title: data.title, status: 'leaping', links: [], target: null,
+ thought: thoughtFor('leaping', { text: lead.text, host: hostOf(lead.url) }),
+ });
+ await this.idle(this.opts.leapMs);
+ }
+ return null;
+ }
+
+ // Bring the chosen link on screen, push the walking step with its rect as the
+ // target, give the avatar time to walk there, then hop.
+ async walkTo(link, pageUrl, pageHost) {
+ const ok = await this.page.evaluate(revealLink, link.i).catch(() => false);
+ if (!ok) return null;
+ const measured = await this.page.evaluate(visibleLinks, 400).catch(() => null);
+ if (!measured) return null;
+ let list = measured.links;
+ let idx = list.findIndex((l) => l.i === link.i);
+ if (idx < 0) return null;
+ if (list.length > STEP_LINKS) {
+ const tgt = list[idx];
+ list = [tgt, ...list.filter((l) => l !== tgt).slice(0, STEP_LINKS - 1)];
+ idx = 0;
+ }
+ const frame = link.visible ? null : await this.screenshot();
+ const host = hostOf(link.href);
+ this.status = 'walking';
+ await this.pushStep({
+ url: pageUrl, title: '', status: 'walking', scrollY: measured.scrollY,
+ links: list.map(({ x, y, w, h, t }) => ({ x, y, w, h, t })), target: idx, nextUrl: link.href,
+ thought: thoughtFor('walking', { text: link.text, host, offsite: host !== pageHost }),
+ }, frame ? { frame } : {});
+ await this.idle(this.opts.walkMs);
+ if (this.stopped) return null;
+ await this.pushStep({
+ url: pageUrl, title: '', status: 'leaping', scrollY: measured.scrollY,
+ links: list.map(({ x, y, w, h, t }) => ({ x, y, w, h, t })), target: idx, nextUrl: link.href,
+ thought: thoughtFor('walking', { text: link.text, host, offsite: host !== pageHost }),
+ });
+ await this.idle(this.opts.leapMs);
+ this.frontier.drop(link.href);
+ return { url: link.href, score: link.score, from: pageUrl, text: link.text };
+ }
+}
diff --git a/workers/agent-crawler/extract.js b/workers/agent-crawler/extract.js
new file mode 100644
index 000000000..e9be4ed8c
--- /dev/null
+++ b/workers/agent-crawler/extract.js
@@ -0,0 +1,122 @@
+// What the crawler reads off a live page. Every function exported here runs
+// INSIDE the page via page.evaluate, so each one must be self-contained: no
+// closures over module scope, no imports.
+
+// Read the page: title, cleaned main text, and every followable anchor with its
+// on-screen rectangle. Anchors get a data-tws-crawl index so the crawler can
+// scroll a chosen one into view and re-measure it without a fragile selector.
+export function readPage(maxLinks) {
+ const vw = window.innerWidth;
+ const vh = window.innerHeight;
+
+ const pickRoot = () => {
+ const candidates = [...document.querySelectorAll('main, article, [role="main"], #content, #main, .content, .post, .article')];
+ let best = null;
+ let bestLen = 0;
+ for (const el of candidates) {
+ const len = (el.innerText || '').length;
+ if (len > bestLen) { best = el; bestLen = len; }
+ }
+ return best && bestLen > 600 ? best : document.body;
+ };
+
+ const cleanText = (root) => {
+ if (!root) return '';
+ const clone = root.cloneNode(true);
+ clone.querySelectorAll('script, style, noscript, svg, canvas, iframe, form, nav, footer, header, aside, button, select, [aria-hidden="true"], [hidden]').forEach((n) => n.remove());
+ // innerText on a detached clone ignores layout, so fall back to textContent
+ // with block elements turned into line breaks.
+ clone.querySelectorAll('p, div, li, h1, h2, h3, h4, h5, h6, tr, br, section, article, blockquote, pre').forEach((n) => n.append('\n'));
+ return (clone.textContent || '')
+ .split('\n')
+ .map((l) => l.replace(/\s+/g, ' ').trim())
+ .filter((l) => l.length > 1)
+ .join('\n');
+ };
+
+ const isShown = (el) => {
+ const cs = getComputedStyle(el);
+ return cs.visibility !== 'hidden' && cs.display !== 'none' && Number(cs.opacity) > 0.05;
+ };
+
+ const anchors = [...document.querySelectorAll('a[href]')];
+ const links = [];
+ for (let i = 0; i < anchors.length && links.length < maxLinks; i++) {
+ const a = anchors[i];
+ const href = a.href;
+ if (!href || !/^https?:/i.test(href)) continue;
+ a.setAttribute('data-tws-crawl', String(i));
+ const r = a.getBoundingClientRect();
+ const sized = r.width >= 6 && r.height >= 6;
+ const onScreen = sized && r.bottom > 0 && r.right > 0 && r.top < vh && r.left < vw;
+ links.push({
+ i,
+ href,
+ text: (a.innerText || a.getAttribute('aria-label') || a.title || '').replace(/\s+/g, ' ').trim().slice(0, 140),
+ visible: onScreen && isShown(a),
+ x: r.left / vw,
+ y: r.top / vh,
+ w: r.width / vw,
+ h: r.height / vh,
+ });
+ }
+
+ const docH = Math.max(document.documentElement.scrollHeight, document.body?.scrollHeight || 0, vh);
+ return {
+ title: (document.title || document.querySelector('h1')?.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 200),
+ lang: document.documentElement.lang || '',
+ text: cleanText(pickRoot()).slice(0, 120000),
+ links,
+ scrollY: docH > vh ? window.scrollY / (docH - vh) : 0,
+ };
+}
+
+// Bring one anchor to the middle of the screen. Returns false when the anchor
+// vanished (single-page apps re-render).
+export function revealLink(index) {
+ const a = document.querySelector(`a[data-tws-crawl="${index}"]`);
+ if (!a) return false;
+ a.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });
+ return true;
+}
+
+// Re-measure only what is on screen right now, in the shape the live step
+// carries: normalized rects clipped to the viewport.
+export function visibleLinks(maxLinks) {
+ const vw = window.innerWidth;
+ const vh = window.innerHeight;
+ const out = [];
+ for (const a of document.querySelectorAll('a[data-tws-crawl]')) {
+ if (out.length >= maxLinks) break;
+ const r = a.getBoundingClientRect();
+ if (r.width < 6 || r.height < 6 || r.bottom <= 0 || r.right <= 0 || r.top >= vh || r.left >= vw) continue;
+ const cs = getComputedStyle(a);
+ if (cs.visibility === 'hidden' || cs.display === 'none' || Number(cs.opacity) <= 0.05) continue;
+ const x = Math.max(0, r.left);
+ const y = Math.max(0, r.top);
+ out.push({
+ i: Number(a.getAttribute('data-tws-crawl')),
+ href: a.href,
+ t: (a.innerText || a.getAttribute('aria-label') || '').replace(/\s+/g, ' ').trim().slice(0, 80),
+ x: x / vw,
+ y: y / vh,
+ w: (Math.min(vw, r.right) - x) / vw,
+ h: (Math.min(vh, r.bottom) - y) / vh,
+ });
+ }
+ const docH = Math.max(document.documentElement.scrollHeight, document.body?.scrollHeight || 0, vh);
+ return { links: out, scrollY: docH > vh ? window.scrollY / (docH - vh) : 0 };
+}
+
+// Consent walls cover the very page viewers came to watch. Hide the common
+// vendors' overlays (never click them: the crawler accepts nothing on anyone's
+// behalf). Injected as an init script on every document.
+export const CONSENT_CSS = [
+ '#onetrust-banner-sdk', '#onetrust-consent-sdk', '.onetrust-pc-dark-filter', '#CybotCookiebotDialog',
+ '#CybotCookiebotDialogBodyUnderlay', '.fc-consent-root', '.qc-cmp2-container', '#qc-cmp2-container',
+ '#truste-consent-track', '.truste_box_overlay', '.cc-window', '.cc-banner', '#cookie-banner', '#cookieBanner',
+ '#cookie-notice', '.cookie-notice', '#cookie-consent', '.cookie-consent', '#gdpr-banner', '.gdpr-banner',
+ '[id^="sp_message_container"]', '.sp_veil', '#usercentrics-root', '#didomi-host', '.didomi-popup-backdrop',
+ '#iubenda-cs-banner', '.osano-cm-window', '#hs-eu-cookie-confirmation',
+].join(',') + '{display:none !important;visibility:hidden !important}'
+ + 'html,body{overflow:auto !important}';
diff --git a/workers/agent-crawler/frontier.js b/workers/agent-crawler/frontier.js
new file mode 100644
index 000000000..b751fd92b
--- /dev/null
+++ b/workers/agent-crawler/frontier.js
@@ -0,0 +1,241 @@
+// Link choice for one crawler: which page does the agent walk to next?
+//
+// Pure on purpose (no browser, no network) so the root test suite can pin the
+// behavior: tests/agent-crawler-frontier.test.js. The crawler hands every link it
+// measured on a page to scoreLink(), keeps the best candidates in a Frontier, and
+// walks to the winner when it is on screen or leaps to it when it is not.
+
+const STOPWORDS = new Set(
+ ('a an and are as at be but by for from has have how in into is it its of on or that the their this to was were what '
+ + 'when where which who why will with about after all also any can could did does each more most new not now our out '
+ + 'over so some such than them then there these they through up use used using very was way we you your').split(' '),
+);
+
+// Links a reading agent should never follow: account walls, carts, share
+// intents, and anything that downloads instead of rendering.
+const SKIP_PATH = /\/(login|log-in|signin|sign-in|signup|sign-up|register|logout|log-out|cart|checkout|basket|account|my-account|subscribe|password|auth|oauth|wp-admin|wp-login)(\/|$|\?|\.)/i;
+const SKIP_HOST = /(^|\.)(facebook\.com|instagram\.com|tiktok\.com|x\.com|twitter\.com|t\.co|linkedin\.com|pinterest\.com|reddit\.com\/submit|accounts\.google\.com|doubleclick\.net|googleadservices\.com)$/i;
+const SKIP_EXT = /\.(pdf|zip|gz|tgz|rar|7z|exe|dmg|msi|iso|apk|deb|rpm|jpg|jpeg|png|gif|webp|avif|svg|ico|bmp|tiff?|mp3|mp4|m4a|wav|ogg|webm|mov|avi|mkv|woff2?|ttf|otf|css|js|json|xml|rss|atom|csv|xlsx?|docx?|pptx?)$/i;
+const SHARE = /(sharer|share\?|intent\/tweet|shareArticle|mailto:|javascript:|tel:|sms:|whatsapp:)/i;
+const BOILERPLATE = /^(privacy|privacy policy|terms|terms of (use|service)|cookies?|cookie (policy|settings)|contact( us)?|careers|jobs|advertise|sitemap|accessibility|imprint|legal|help|faq|log ?in|sign ?(in|up)|subscribe|donate|skip to (main )?content|menu|search|home)$/i;
+
+export const MAX_FRONTIER = 400;
+export const MAX_PAGES_PER_DOMAIN = 30;
+
+// Lowercase content words with a light suffix strip, so "crawlers" in a topic
+// matches "crawler" in an anchor and "learning" matches "learn".
+export function stem(word) {
+ let w = word.toLowerCase();
+ if (w.length > 5 && w.endsWith('ing')) w = w.slice(0, -3);
+ else if (w.length > 4 && w.endsWith('ies')) w = `${w.slice(0, -3)}y`;
+ else if (w.length > 4 && w.endsWith('ed')) w = w.slice(0, -2);
+ else if (w.length > 3 && w.endsWith('s') && !w.endsWith('ss')) w = w.slice(0, -1);
+ return w;
+}
+
+export function tokenize(text) {
+ return String(text || '')
+ .toLowerCase()
+ .split(/[^\p{L}\p{N}]+/u)
+ .filter((w) => w.length >= 3 && !STOPWORDS.has(w))
+ .map(stem);
+}
+
+export function topicTerms(topic) {
+ return [...new Set(tokenize(topic))].slice(0, 16);
+}
+
+export function hostOf(url) {
+ try { return new URL(url).hostname.toLowerCase().replace(/^www\./, ''); } catch { return ''; }
+}
+
+// Strip the fragment and tracking tags so the same article reached from two
+// pages counts as one visit. Mirrors normalizeUrl in api/_lib/crawl.js closely
+// enough for dedupe; the API re-normalizes authoritatively.
+export function canonical(url) {
+ let u;
+ try { u = new URL(url); } catch { return null; }
+ if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
+ if (u.username || u.password) return null;
+ u.hash = '';
+ for (const k of [...u.searchParams.keys()]) {
+ if (/^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_[a-z]+|ref|ref_src|igshid|si)$/i.test(k)) u.searchParams.delete(k);
+ }
+ u.searchParams.sort();
+ if (u.pathname.length > 1 && u.pathname.endsWith('/')) u.pathname = u.pathname.replace(/\/+$/, '');
+ return u.href;
+}
+
+export function isCrawlable(url) {
+ if (!url || url.length > 400 || SHARE.test(url)) return false;
+ let u;
+ try { u = new URL(url); } catch { return false; }
+ if (u.protocol !== 'https:' && u.protocol !== 'http:') return false;
+ if (SKIP_HOST.test(u.hostname) || SKIP_HOST.test(`${u.hostname}${u.pathname}`)) return false;
+ if (SKIP_PATH.test(u.pathname) || SKIP_EXT.test(u.pathname)) return false;
+ return true;
+}
+
+// Fraction of topic terms present in a token list, 0..1.
+export function coverage(terms, tokens) {
+ if (!terms.length) return 0;
+ const set = tokens instanceof Set ? tokens : new Set(tokens);
+ let hit = 0;
+ for (const t of terms) if (set.has(t)) hit += 1;
+ return hit / terms.length;
+}
+
+// How close is a page to the topic? Title hits count double, and body density
+// (term mentions per thousand words, saturating) separates a passing mention
+// from a page that is about the thing.
+export function relevance(terms, title, text) {
+ if (!terms.length) return 0;
+ const titleCov = coverage(terms, tokenize(title));
+ const words = tokenize(String(text || '').slice(0, 40_000));
+ if (!words.length) return Math.min(1, titleCov * 0.6);
+ const termSet = new Set(terms);
+ let mentions = 0;
+ for (const w of words) if (termSet.has(w)) mentions += 1;
+ const density = Math.min(1, (mentions / words.length) * 1000 / 25);
+ const bodyCov = coverage(terms, words);
+ return Math.round(Math.min(1, titleCov * 0.4 + bodyCov * 0.35 + density * 0.25) * 1000) / 1000;
+}
+
+// Score one link. ctx: { terms, fromHost, domainCounts: Map, visited: Set, rand }.
+// Returns -Infinity for links that must never be followed.
+export function scoreLink(link, ctx) {
+ const url = canonical(link.href);
+ if (!url || !isCrawlable(url) || ctx.visited.has(url)) return -Infinity;
+ const host = hostOf(url);
+ const seen = ctx.domainCounts.get(host) || 0;
+ if (seen >= MAX_PAGES_PER_DOMAIN) return -Infinity;
+
+ const text = String(link.text || '').trim();
+ if (BOILERPLATE.test(text)) return -Infinity;
+
+ const textTokens = tokenize(text);
+ let pathTokens = [];
+ try { pathTokens = tokenize(decodeURIComponent(new URL(url).pathname.replace(/[-_/.]+/g, ' '))); } catch { pathTokens = []; }
+
+ const textHits = ctx.terms.filter((t) => textTokens.includes(t)).length;
+ const pathHits = ctx.terms.filter((t) => pathTokens.includes(t)).length;
+ let score = textHits * 3 + pathHits * 1.5;
+
+ // Descriptive anchors beat "click here" and bare icons.
+ if (text.length >= 12 && text.length <= 120) score += 0.6;
+ else if (!text) score -= 0.8;
+
+ const sameHost = host === ctx.fromHost;
+ if (sameHost) score += 0.4;
+ else if (seen === 0 && (textHits || pathHits)) score += 1.2; // a relevant new site widens the corpus
+ else if (seen === 0) score += 0.3;
+ score -= seen * 0.08; // wear off a domain gradually
+
+ // Prefer article-shaped paths over hubs and query soups.
+ const depth = pathTokens.length;
+ if (depth >= 2 && depth <= 10) score += 0.3;
+ if (url.includes('?') && url.split('&').length > 3) score -= 0.6;
+
+ if (link.visible) score += 0.25; // the agent can walk to it on this screen
+ return score + (ctx.rand ? ctx.rand() * 0.35 : 0);
+}
+
+// A bounded best-first queue of URLs, deduped by canonical URL.
+export class Frontier {
+ constructor(max = MAX_FRONTIER) {
+ this.max = max;
+ this.items = new Map(); // url -> { url, score, from, text }
+ }
+
+ get size() { return this.items.size; }
+
+ add(url, score, from = null, text = '') {
+ const key = canonical(url);
+ if (!key || !Number.isFinite(score)) return false;
+ const prev = this.items.get(key);
+ if (prev && prev.score >= score) return false;
+ this.items.set(key, { url: key, score, from, text });
+ if (this.items.size > this.max) this.trim();
+ return true;
+ }
+
+ trim() {
+ const sorted = [...this.items.values()].sort((a, b) => b.score - a.score).slice(0, this.max);
+ this.items = new Map(sorted.map((i) => [i.url, i]));
+ }
+
+ // Highest-scoring entry that passes the filter (e.g. not visited, not cooling
+ // down). Entries that fail are left in place for a later turn.
+ pop(accept = () => true) {
+ let best = null;
+ for (const item of this.items.values()) {
+ if (!accept(item)) continue;
+ if (!best || item.score > best.score) best = item;
+ }
+ if (best) this.items.delete(best.url);
+ return best;
+ }
+
+ drop(url) { this.items.delete(canonical(url)); }
+
+ // Decay everything a little each turn so stale hopes from fifty pages ago
+ // lose to fresh leads from the page the agent is actually standing on.
+ decay(factor = 0.97) {
+ for (const item of this.items.values()) item.score *= factor;
+ }
+}
+
+// Extractive gist: the first few real sentences, preferring ones that mention
+// the topic. No model call, so it is free, deterministic, and never invents.
+export function gistOf(text, terms, max = 480) {
+ // Lines are block boundaries (readPage puts one per paragraph), so a menu
+ // crumb on its own line never glues onto the first real sentence.
+ const sentences = String(text || '')
+ .split(/\n+/)
+ .flatMap((line) => line.replace(/\s+/g, ' ').split(/(?<=[.!?])\s+(?=[A-Z0-9"“(])/))
+ .map((s) => s.trim())
+ .filter((s) => s.length >= 40 && s.length <= 400 && /[a-z]/.test(s) && (s.match(/\s/g) || []).length >= 5)
+ .slice(0, 40);
+ if (!sentences.length) return '';
+ const termSet = new Set(terms);
+ const ranked = sentences.map((s, i) => {
+ const hits = tokenize(s).filter((t) => termSet.has(t)).length;
+ return { s, i, rank: hits * 2 - i * 0.15 };
+ });
+ const picked = ranked.sort((a, b) => b.rank - a.rank).slice(0, 3).sort((a, b) => a.i - b.i);
+ let out = '';
+ for (const { s } of picked) {
+ if ((out ? out.length + 1 : 0) + s.length > max) break;
+ out = out ? `${out} ${s}` : s;
+ }
+ return out || sentences[0].slice(0, max);
+}
+
+// What the agent "says" on its card. Deterministic phrasing from what it is
+// really doing; no model is asked to narrate.
+export function thoughtFor(kind, d = {}) {
+ const quote = (s, n = 70) => {
+ const t = String(s || '').replace(/\s+/g, ' ').trim();
+ return t.length > n ? `${t.slice(0, n - 1)}…` : t;
+ };
+ switch (kind) {
+ case 'reading':
+ return d.relevance >= 0.35
+ ? `Reading "${quote(d.title)}" on ${d.domain}. This is squarely about ${d.topic}.`
+ : d.relevance >= 0.12
+ ? `Reading "${quote(d.title)}" on ${d.domain}. Some of this touches ${d.topic}.`
+ : `Passing through ${d.domain}. Not much about ${d.topic} here.`;
+ case 'walking':
+ return d.text
+ ? `Walking to "${quote(d.text, 60)}"${d.offsite ? `, off to ${d.host}` : ''}.`
+ : `Walking to a link${d.offsite ? ` on ${d.host}` : ''}.`;
+ case 'leaping':
+ return d.text ? `Leaping to "${quote(d.text, 60)}" from my list.` : `Leaping to ${d.host} from my list.`;
+ case 'blocked':
+ return `${d.host} ${d.reason || 'turned me away'}. Finding another way.`;
+ case 'resting':
+ return d.reason || 'Out of leads. Looking for new start pages.';
+ default:
+ return '';
+ }
+}
diff --git a/workers/agent-crawler/index.js b/workers/agent-crawler/index.js
new file mode 100644
index 000000000..0ec13c760
--- /dev/null
+++ b/workers/agent-crawler/index.js
@@ -0,0 +1,204 @@
+// agent-crawler: three.ws agents reading the open web in real browsers.
+//
+// The worker behind /crawl. It polls the roster of agents whose owners sent them
+// out (GET /api/crawl/roster), keeps up to CRAWLERS of them walking at once (one
+// shared Chromium, one isolated context per agent), and pushes every step and
+// every finished page back through POST /api/crawl/push. The API fans steps out
+// to viewers live and files pages into the open corpus. Doc: docs/crawl.md.
+//
+// When more agents are enrolled than there are slots, each walks a shift of
+// SHIFT_MS and then yields to the agent that has waited longest (the roster is
+// ordered by last read, oldest first), so every enrolled agent gets its turn.
+//
+// Required env:
+// CRAWL_WORKER_SECRET shared secret; must match the API's value.
+// Optional env:
+// BASE_URL=https://three.ws CRAWLERS=8 SHIFT_MS=1200000 ROSTER_MS=30000
+// READ_MS=3500 WALK_MS=3200 LEAP_MS=900 DOMAIN_GAP_MS=4000 NAV_TIMEOUT_MS=20000
+// JPEG_QUALITY=55
+// PORT when set, bind the liveness endpoint (Cloud Run sets it).
+
+import http from 'node:http';
+import { chromium } from 'playwright';
+import { Crawler, NotEnrolledError } from './crawler.js';
+
+const BASE_URL = (process.env.BASE_URL || 'https://three.ws').replace(/\/$/, '');
+const SECRET = process.env.CRAWL_WORKER_SECRET || '';
+const CRAWLERS = Math.max(1, Number(process.env.CRAWLERS || 8));
+const SHIFT_MS = Number(process.env.SHIFT_MS || 20 * 60_000);
+const ROSTER_MS = Number(process.env.ROSTER_MS || 30_000);
+const OPTS = {
+ readMs: Number(process.env.READ_MS || 3500),
+ walkMs: Number(process.env.WALK_MS || 3200),
+ leapMs: Number(process.env.LEAP_MS || 900),
+ domainGapMs: Number(process.env.DOMAIN_GAP_MS || 4000),
+ navTimeoutMs: Number(process.env.NAV_TIMEOUT_MS || 20_000),
+ jpegQuality: Number(process.env.JPEG_QUALITY || 55),
+};
+
+const log = (msg) => console.log(`${new Date().toISOString()} ${msg}`);
+
+if (SECRET.length < 16) {
+ console.error('CRAWL_WORKER_SECRET is missing or shorter than 16 characters; the API would refuse every call.');
+ process.exit(1);
+}
+
+// ── API client ──────────────────────────────────────────────────────────────
+const api = {
+ async roster(limit) {
+ const res = await fetch(`${BASE_URL}/api/crawl/roster?limit=${limit}`, {
+ headers: { authorization: `Bearer ${SECRET}` },
+ signal: AbortSignal.timeout(15_000),
+ });
+ if (!res.ok) throw new Error(`roster ${res.status}`);
+ return (await res.json()).crawlers || [];
+ },
+
+ async recentPages(agentId, limit) {
+ const res = await fetch(`${BASE_URL}/api/crawl/pages?agent=${agentId}&limit=${limit}`, {
+ signal: AbortSignal.timeout(15_000),
+ });
+ if (!res.ok) throw new Error(`pages ${res.status}`);
+ return (await res.json()).pages || [];
+ },
+
+ // One retry on a network blip or 5xx; a 410 means the owner called the agent
+ // home, which ends that crawler. Anything else is logged and the walk goes on.
+ async push(body) {
+ const payload = JSON.stringify(body);
+ for (let attempt = 0; attempt < 2; attempt++) {
+ try {
+ const res = await fetch(`${BASE_URL}/api/crawl/push`, {
+ method: 'POST',
+ headers: { authorization: `Bearer ${SECRET}`, 'content-type': 'application/json' },
+ body: payload,
+ signal: AbortSignal.timeout(20_000),
+ });
+ if (res.status === 410) throw new NotEnrolledError('mission ended');
+ if (res.ok) return await res.json();
+ if (res.status < 500) {
+ log(`push rejected ${res.status}: ${(await res.text()).slice(0, 200)}`);
+ return null;
+ }
+ } catch (err) {
+ if (err instanceof NotEnrolledError) throw err;
+ if (attempt) log(`push failed: ${err?.message || err}`);
+ }
+ await new Promise((r) => { setTimeout(r, 1500); });
+ }
+ return null;
+ },
+};
+
+// ── browser ─────────────────────────────────────────────────────────────────
+let browser = null;
+
+async function ensureBrowser() {
+ if (browser?.isConnected()) return browser;
+ browser = await chromium.launch({
+ headless: true,
+ args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--mute-audio'],
+ });
+ browser.on('disconnected', () => {
+ log('chromium disconnected; crawlers will restart on the next roster tick');
+ for (const c of running.values()) c.stopped = true;
+ running.clear();
+ });
+ log('chromium launched');
+ return browser;
+}
+
+// ── orchestration ───────────────────────────────────────────────────────────
+const running = new Map(); // agentId -> Crawler
+let lastRosterError = null;
+let lastRosterAt = 0;
+
+function startCrawler(mission) {
+ const crawler = new Crawler({ mission, browser, api, log, opts: OPTS });
+ running.set(mission.agentId, crawler);
+ log(`out reading: ${mission.name} on "${mission.topic}"`);
+ crawler.start().finally(() => {
+ if (running.get(mission.agentId) === crawler) running.delete(mission.agentId);
+ });
+}
+
+async function tick() {
+ let roster;
+ try {
+ roster = await api.roster(Math.max(CRAWLERS * 3, 24));
+ lastRosterError = null;
+ lastRosterAt = Date.now();
+ } catch (err) {
+ lastRosterError = err?.message || String(err);
+ log(`roster unavailable: ${lastRosterError}`);
+ return;
+ }
+ const enrolled = new Map(roster.map((m) => [m.agentId, m]));
+
+ // Called home, or the topic changed.
+ for (const [id, c] of running) {
+ const m = enrolled.get(id);
+ if (!m) {
+ log(`called home: ${c.name}`);
+ running.delete(id);
+ c.stop();
+ } else {
+ c.updateMission(m);
+ }
+ }
+
+ const waiting = roster.filter((m) => !running.has(m.agentId));
+ // Shift change: if someone is waiting, the longest-running crawler past its
+ // shift steps aside.
+ if (waiting.length && running.size >= CRAWLERS) {
+ const overdue = [...running.values()]
+ .filter((c) => Date.now() - c.startedAt > SHIFT_MS)
+ .sort((a, b) => a.startedAt - b.startedAt);
+ for (const c of overdue.slice(0, waiting.length)) {
+ log(`shift over: ${c.name}`);
+ running.delete(c.agentId);
+ c.stop();
+ }
+ }
+
+ if (!waiting.length || running.size >= CRAWLERS) return;
+ await ensureBrowser();
+ for (const m of waiting) {
+ if (running.size >= CRAWLERS) break;
+ startCrawler(m);
+ }
+}
+
+async function loop() {
+ for (;;) {
+ await tick().catch((err) => log(`tick failed: ${err?.stack || err}`));
+ await new Promise((r) => { setTimeout(r, ROSTER_MS); });
+ }
+}
+
+// ── liveness ────────────────────────────────────────────────────────────────
+if (process.env.PORT) {
+ http.createServer((req, res) => {
+ res.writeHead(200, { 'content-type': 'application/json' });
+ res.end(JSON.stringify({
+ ok: true,
+ crawlers: [...running.values()].map((c) => c.summary()),
+ slots: CRAWLERS,
+ browser: Boolean(browser?.isConnected()),
+ lastRosterAt,
+ lastRosterError,
+ }));
+ }).listen(Number(process.env.PORT), () => log(`liveness on :${process.env.PORT}`));
+}
+
+async function shutdown(signal) {
+ log(`${signal}: calling every crawler home`);
+ await Promise.allSettled([...running.values()].map((c) => c.stop()));
+ await browser?.close().catch(() => {});
+ process.exit(0);
+}
+process.on('SIGTERM', () => { shutdown('SIGTERM'); });
+process.on('SIGINT', () => { shutdown('SIGINT'); });
+
+log(`agent-crawler: ${CRAWLERS} slots against ${BASE_URL}`);
+loop();
diff --git a/workers/agent-crawler/net.js b/workers/agent-crawler/net.js
new file mode 100644
index 000000000..b1974e0d5
--- /dev/null
+++ b/workers/agent-crawler/net.js
@@ -0,0 +1,116 @@
+// Network fences for a browser that goes wherever links lead.
+//
+// guardHost(): every request the crawler's pages make is resolved first and
+// refused when ANY address it resolves to is private, loopback, link-local or
+// reserved. That is what keeps a hostile page (or a mission seed pointing at an
+// internal name) from turning the crawler into a probe of the network it runs in.
+//
+// Robots: robots.txt is fetched once per origin and honored, including
+// Crawl-delay, under the crawler's own product token.
+
+import dns from 'node:dns/promises';
+import net from 'node:net';
+import robotsParser from 'robots-parser';
+
+export const BOT_TOKEN = 'three.ws-crawler';
+export const USER_AGENT =
+ `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 ${BOT_TOKEN}/1.0 (+https://three.ws/crawl)`;
+
+const blocked = new net.BlockList();
+for (const [addr, prefix] of [
+ ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8], ['169.254.0.0', 16], ['172.16.0.0', 12],
+ ['192.0.0.0', 24], ['192.0.2.0', 24], ['192.168.0.0', 16], ['198.18.0.0', 15], ['198.51.100.0', 24],
+ ['203.0.113.0', 24], ['224.0.0.0', 4], ['240.0.0.0', 4],
+]) blocked.addSubnet(addr, prefix, 'ipv4');
+for (const [addr, prefix] of [
+ ['::', 128], ['::1', 128], ['::ffff:0:0', 96], ['64:ff9b::', 96], ['100::', 64], ['2001:db8::', 32],
+ ['fc00::', 7], ['fe80::', 10], ['ff00::', 8],
+]) blocked.addSubnet(addr, prefix, 'ipv6');
+
+export function isBlockedAddress(ip) {
+ const family = net.isIP(ip);
+ if (!family) return true;
+ return blocked.check(ip, family === 4 ? 'ipv4' : 'ipv6');
+}
+
+const INTERNAL_NAME = /(^|\.)(localhost|local|internal|intranet|lan|home|corp)$|^metadata(\.google\.internal)?$/i;
+const hostCache = new Map(); // host -> { ok, at }
+const HOST_TTL_MS = 10 * 60_000;
+
+export async function guardHost(hostname) {
+ const host = String(hostname || '').toLowerCase().replace(/^\[|\]$/g, '');
+ if (!host || INTERNAL_NAME.test(host)) return false;
+ if (net.isIP(host)) return !isBlockedAddress(host);
+ const hit = hostCache.get(host);
+ if (hit && Date.now() - hit.at < HOST_TTL_MS) return hit.ok;
+ let ok = false;
+ try {
+ const addrs = await dns.lookup(host, { all: true, verbatim: true });
+ ok = addrs.length > 0 && addrs.every((a) => !isBlockedAddress(a.address));
+ } catch {
+ ok = false;
+ }
+ hostCache.set(host, { ok, at: Date.now() });
+ if (hostCache.size > 5000) hostCache.delete(hostCache.keys().next().value);
+ return ok;
+}
+
+// fetch() for the crawler's own side requests (robots.txt, search reseeds),
+// with the same host fence and a hard timeout.
+export async function guardedFetch(url, { timeoutMs = 8000, headers = {} } = {}) {
+ const u = new URL(url);
+ if (!(await guardHost(u.hostname))) throw new Error(`refused private host ${u.hostname}`);
+ return fetch(u, {
+ headers: { 'user-agent': USER_AGENT, accept: '*/*', ...headers },
+ redirect: 'follow',
+ signal: AbortSignal.timeout(timeoutMs),
+ });
+}
+
+const robotsCache = new Map(); // origin -> { robots, at }
+const ROBOTS_TTL_MS = 60 * 60_000;
+
+async function robotsFor(origin) {
+ const hit = robotsCache.get(origin);
+ if (hit && Date.now() - hit.at < ROBOTS_TTL_MS) return hit.robots;
+ const robotsUrl = `${origin}/robots.txt`;
+ let body = '';
+ try {
+ const res = await guardedFetch(robotsUrl, { timeoutMs: 6000, headers: { accept: 'text/plain' } });
+ // RFC 9309: 4xx means no rules; 5xx or a network failure means "assume
+ // disallowed for now" so a struggling server is not crawled harder.
+ if (res.status >= 500) body = 'User-agent: *\nDisallow: /';
+ else if (res.ok) body = (await res.text()).slice(0, 500_000);
+ } catch {
+ body = 'User-agent: *\nDisallow: /';
+ }
+ const robots = robotsParser(robotsUrl, body);
+ robotsCache.set(origin, { robots, at: Date.now() });
+ if (robotsCache.size > 3000) robotsCache.delete(robotsCache.keys().next().value);
+ return robots;
+}
+
+// { allowed, delayMs } for one URL under the crawler's token.
+export async function robotsCheck(url) {
+ const u = new URL(url);
+ const robots = await robotsFor(u.origin);
+ const allowed = robots.isAllowed(u.href, BOT_TOKEN) !== false;
+ const delay = Number(robots.getCrawlDelay(BOT_TOKEN));
+ return { allowed, delayMs: Number.isFinite(delay) && delay > 0 ? Math.min(delay, 30) * 1000 : 0 };
+}
+
+// Per-domain politeness shared by every crawler in the process: at most one
+// request to a domain every gapMs (or its robots Crawl-delay, if longer).
+const domainNext = new Map();
+
+export function domainReadyIn(host) {
+ return Math.max(0, (domainNext.get(host) || 0) - Date.now());
+}
+
+export function claimDomain(host, gapMs) {
+ domainNext.set(host, Date.now() + gapMs);
+ if (domainNext.size > 10_000) {
+ const now = Date.now();
+ for (const [h, t] of domainNext) if (t < now) domainNext.delete(h);
+ }
+}
diff --git a/workers/agent-crawler/package-lock.json b/workers/agent-crawler/package-lock.json
new file mode 100644
index 000000000..3a056e42a
--- /dev/null
+++ b/workers/agent-crawler/package-lock.json
@@ -0,0 +1,72 @@
+{
+ "name": "agent-crawler",
+ "version": "0.1.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "agent-crawler",
+ "version": "0.1.0",
+ "dependencies": {
+ "playwright": "1.61.1",
+ "robots-parser": "^3.0.1"
+ },
+ "engines": {
+ "node": ">=20"
+ }
+ },
+ "node_modules/fsevents": {
+ "version": "2.3.2",
+ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+ "hasInstallScript": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+ }
+ },
+ "node_modules/playwright": {
+ "version": "1.61.1",
+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz",
+ "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright-core": "1.61.1"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "optionalDependencies": {
+ "fsevents": "2.3.2"
+ }
+ },
+ "node_modules/playwright-core": {
+ "version": "1.61.1",
+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz",
+ "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==",
+ "license": "Apache-2.0",
+ "bin": {
+ "playwright-core": "cli.js"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/robots-parser": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz",
+ "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=10.0.0"
+ }
+ }
+ }
+}
diff --git a/workers/agent-crawler/package.json b/workers/agent-crawler/package.json
new file mode 100644
index 000000000..4cdd05560
--- /dev/null
+++ b/workers/agent-crawler/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "agent-crawler",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "index.js",
+ "scripts": {
+ "start": "node index.js"
+ },
+ "engines": { "node": ">=20" },
+ "dependencies": {
+ "playwright": "1.61.1",
+ "robots-parser": "^3.0.1"
+ }
+}
diff --git a/workers/agent-crawler/seeds.js b/workers/agent-crawler/seeds.js
new file mode 100644
index 000000000..341a2f181
--- /dev/null
+++ b/workers/agent-crawler/seeds.js
@@ -0,0 +1,69 @@
+// Where an agent starts when its owner gave no start pages, or when it has
+// walked itself into a dead end: real public search, tried in order until one
+// answers. Each source is a free, keyless public endpoint.
+
+import { guardedFetch } from './net.js';
+
+// DuckDuckGo's HTML endpoint. Result anchors carry the target in the uddg
+// parameter of a /l/ redirect. It sometimes answers bots with a 202 challenge
+// page; that simply yields nothing and the next source takes over.
+async function duckduckgo(topic) {
+ const res = await guardedFetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(topic)}`, {
+ headers: { accept: 'text/html' },
+ });
+ if (res.status !== 200) return [];
+ const html = await res.text();
+ const out = [];
+ for (const m of html.matchAll(/class="result__a"[^>]*href="([^"]+)"/g)) {
+ const href = m[1].replace(/&amp;/g, '&');
+ try {
+ const u = new URL(href, 'https://duckduckgo.com');
+ const target = u.searchParams.get('uddg') || (u.hostname.endsWith('duckduckgo.com') ? null : u.href);
+ if (target && !/duckduckgo\.com\/y\.js/.test(target)) out.push(target);
+ } catch { /* skip malformed result */ }
+ }
+ return out;
+}
+
+// Hacker News search (Algolia): stories carry the outside URL they link to,
+// which is exactly the kind of page a reading agent should land on.
+async function hackernews(topic) {
+ const res = await guardedFetch(
+ `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(topic)}&tags=story&hitsPerPage=20`,
+ { headers: { accept: 'application/json' } },
+ );
+ if (!res.ok) return [];
+ const data = await res.json();
+ return (data.hits || []).map((h) => h.url).filter((u) => typeof u === 'string' && /^https?:\/\//.test(u));
+}
+
+// Wikipedia full-text search: always answers, always has something.
+async function wikipedia(topic) {
+ const res = await guardedFetch(
+ `https://en.wikipedia.org/w/api.php?action=query&list=search&format=json&srlimit=8&srsearch=${encodeURIComponent(topic)}`,
+ { headers: { accept: 'application/json' } },
+ );
+ if (!res.ok) return [];
+ const data = await res.json();
+ return (data.query?.search || []).map((s) => `https://en.wikipedia.org/wiki/${encodeURIComponent(s.title.replace(/ /g, '_'))}`);
+}
+
+export const SOURCES = [
+ { name: 'duckduckgo', fn: duckduckgo },
+ { name: 'hackernews', fn: hackernews },
+ { name: 'wikipedia', fn: wikipedia },
+];
+
+// Gather start pages for a topic. Returns { urls, source } from the first source
+// with results, so the crawler can say where its leads came from.
+export async function searchSeeds(topic, log = () => {}) {
+ for (const { name, fn } of SOURCES) {
+ try {
+ const urls = [...new Set(await fn(topic))].slice(0, 12);
+ if (urls.length) return { urls, source: name };
+ } catch (err) {
+ log(`seed source ${name} failed: ${err?.message || err}`);
+ }
+ }
+ return { urls: [], source: null };
+}