Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(crawl): add the crawl API, its schema and the agent-crawler worker behind the coming /crawl page
nirholas07eb02034bc2
diff --git a/api/_lib/crawl.js b/api/_lib/crawl.jsnew file mode 100644index 000000000..61623c221--- /dev/null+++ b/api/_lib/crawl.js@@ -0,0 +1,182 @@+// The Crawl: pure helpers shared by the API (api/crawl/[action].js), the worker+// (workers/agent-crawler) and the tests. Nothing here touches the network, the+// database or Redis, so every boundary rule (what a pushed step may carry, what a+// mission may name, how a URL is keyed) is unit-testable and identical on both+// sides of the wire.++import { createHash } from 'node:crypto';++export const CRAWL_LIVE_TTL = 75; // seconds a crawler stays "awake" after its last push+export const CRAWL_ACTIVE_WINDOW_MS = 90_000;+export const MAX_LINKS = 48;+export const MAX_TEXT_CHARS = 120_000;+export const GIST_MAX = 480;+export const TITLE_MAX = 200;+export const THOUGHT_MAX = 240;+export const TOPIC_MIN = 2;+export const TOPIC_MAX = 120;+export const MAX_SEEDS = 8;+export const FRAME_B64_MAX = 700_000; // ~520 KB JPEG+export const CHARS_PER_TOKEN = 4;++export const STEP_STATUSES = ['reading', 'walking', 'leaping', 'blocked', 'resting'];++// Tracking parameters that make the same page look like many. Stripped before a+// URL is hashed so the corpus dedupes on content, not on campaign tags.+const TRACKING_PARAMS = /^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_[a-z]+|ref|ref_src|_hs[a-z]+|igshid|si)$/i;++export function normalizeUrl(raw) {+ let u;+ try { u = new URL(String(raw)); } catch { return null; }+ if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;+ if (u.username || u.password) return null;+ u.hash = '';+ u.hostname = u.hostname.toLowerCase().replace(/\.$/, '');+ if ((u.protocol === 'https:' && u.port === '443') || (u.protocol === 'http:' && u.port === '80')) u.port = '';+ for (const k of [...u.searchParams.keys()]) {+ if (TRACKING_PARAMS.test(k)) u.searchParams.delete(k);+ }+ u.searchParams.sort();+ if (u.pathname.length > 1 && u.pathname.endsWith('/')) u.pathname = u.pathname.replace(/\/+$/, '');+ return u.href;+}++export function urlHash(normalized) {+ return createHash('sha256').update(normalized).digest('hex');+}++export function domainOf(url) {+ try {+ return new URL(url).hostname.toLowerCase().replace(/^www\./, '');+ } catch {+ return '';+ }+}++export function estimateTokens(text) {+ return Math.ceil(String(text || '').length / CHARS_PER_TOKEN);+}++const clamp01 = (n) => (Number.isFinite(n) ? Math.min(1, Math.max(0, n)) : null);+const str = (v, max) => (typeof v === 'string' ? v.replace(/\s+/g, ' ').trim().slice(0, max) : '');++// A visible link rectangle, in normalized viewport coordinates (0..1). The page+// renders the avatar walking to these, so a malformed rect must be dropped, never+// passed through to a client that would place a body off-canvas.+export function sanitizeLinkRect(link) {+ if (!link || typeof link !== 'object') return null;+ const x = clamp01(Number(link.x));+ const y = clamp01(Number(link.y));+ const w = clamp01(Number(link.w));+ const h = clamp01(Number(link.h));+ if (x == null || y == null || w == null || h == null || w <= 0 || h <= 0) return null;+ const out = { x, y, w: Math.min(w, 1 - x), h: Math.min(h, 1 - y), t: str(link.t, 80) };+ if (out.w <= 0 || out.h <= 0) return null;+ return out;+}++// The live step: what a crawler is doing right now. Lite on purpose (no image+// bytes): the fleet stream fans it out to every viewer on every change.+export function sanitizeStep(step) {+ if (!step || typeof step !== 'object') return null;+ const url = normalizeUrl(step.url);+ const links = Array.isArray(step.links)+ ? step.links.slice(0, MAX_LINKS).map(sanitizeLinkRect).filter(Boolean)+ : [];+ const t = Number(step.target);+ const target = Number.isInteger(t) && t >= 0 && t < links.length ? t : null;+ const nextUrl = target != null ? normalizeUrl(step.nextUrl) : null;+ const status = STEP_STATUSES.includes(step.status) ? step.status : 'reading';+ const seq = Number(step.seq);+ return {+ url,+ domain: url ? domainOf(url) : '',+ title: str(step.title, TITLE_MAX),+ thought: str(step.thought, THOUGHT_MAX),+ status,+ links,+ target,+ nextUrl,+ seq: Number.isSafeInteger(seq) && seq >= 0 ? seq : 0,+ scrollY: clamp01(Number(step.scrollY)) ?? 0,+ };+}++// A finished page read: the corpus row. Text is bounded here so a runaway+// page cannot push megabytes through the API.+export function sanitizePage(page) {+ if (!page || typeof page !== 'object') return null;+ const url = normalizeUrl(page.url);+ if (!url) return null;+ const text = typeof page.text === 'string' ? page.text.slice(0, MAX_TEXT_CHARS) : '';+ const rel = Number(page.relevance);+ const linksOut = Number(page.linksOut);+ return {+ url,+ hash: urlHash(url),+ domain: domainOf(url),+ title: str(page.title, TITLE_MAX) || null,+ gist: str(page.gist, GIST_MAX) || null,+ text,+ tokens: estimateTokens(text),+ linksOut: Number.isSafeInteger(linksOut) && linksOut >= 0 ? Math.min(linksOut, 100_000) : 0,+ relevance: Number.isFinite(rel) ? Math.min(1, Math.max(0, rel)) : null,+ fromUrl: normalizeUrl(page.fromUrl),+ };+}++export function isFrameB64(s) {+ return typeof s === 'string' && s.length > 64 && s.length <= FRAME_B64_MAX && /^[A-Za-z0-9+/]+=*$/.test(s)+ && s.startsWith('/9j/'); // JPEG magic (FF D8 FF) in base64+}++// Owner-set mission. Seeds must be public http(s) URLs; the worker re-guards every+// navigation against private hosts, this is the first fence at write time.+export function validateMission(body) {+ const topic = str(body?.topic, TOPIC_MAX + 1);+ if (topic.length < TOPIC_MIN || topic.length > TOPIC_MAX) {+ return { error: `topic must be ${TOPIC_MIN} to ${TOPIC_MAX} characters` };+ }+ const rawSeeds = Array.isArray(body?.seeds)+ ? body.seeds+ : typeof body?.seeds === 'string' ? body.seeds.split(/[\s,]+/) : [];+ const seeds = [];+ for (const s of rawSeeds) {+ if (!s || typeof s !== 'string') continue;+ const withScheme = /^https?:\/\//i.test(s.trim()) ? s.trim() : `https://${s.trim()}`;+ const n = normalizeUrl(withScheme);+ if (!n) return { error: `not a web address: ${s.slice(0, 80)}` };+ const host = new URL(n).hostname;+ if (isPrivateHostname(host)) return { error: `not a public address: ${host}` };+ if (!seeds.includes(n)) seeds.push(n);+ if (seeds.length > MAX_SEEDS) return { error: `at most ${MAX_SEEDS} start pages` };+ }+ return { topic, seeds, enabled: body?.enabled !== false };+}++// Static hostname screen (no DNS). Catches literals and the well-known internal+// names; the worker additionally resolves every host before it navigates.+export function isPrivateHostname(host) {+ const h = String(host || '').toLowerCase().replace(/^\[|\]$/g, '');+ if (!h || (!h.includes('.') && !h.includes(':'))) return true;+ if (h === 'localhost' || h.endsWith('.localhost') || h.endsWith('.local') || h.endsWith('.internal')) return true;+ if (h === 'metadata.google.internal' || h === 'metadata') return true;+ const v4 = h.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);+ if (v4) {+ const [a, b] = [Number(v4[1]), Number(v4[2])];+ return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31)+ || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224;+ }+ if (h.includes(':')) {+ return h === '::1' || h === '::' || h.startsWith('fc') || h.startsWith('fd') || h.startsWith('fe80') || h.startsWith('::ffff:');+ }+ return false;+}++// The memory an owner's agent keeps from a page it read, so its chat can recall+// it later ("what have you been reading about?").+export function pageMemory(page, topic) {+ const head = page.title ? `Read "${page.title}"` : 'Read a page';+ const body = page.gist ? `: ${page.gist}` : '.';+ return `${head} (${page.url}) while crawling for ${topic}${body}`.slice(0, 900);+}diff --git a/api/_lib/migrations/20261009230000_agent_crawl.sql b/api/_lib/migrations/20261009230000_agent_crawl.sqlnew file mode 100644index 000000000..8c84b63a2--- /dev/null+++ b/api/_lib/migrations/20261009230000_agent_crawl.sql@@ -0,0 +1,53 @@+-- Migration: the Crawl, agents that read the open web in real browsers.+-- Apply: node scripts/apply-migrations.mjs --apply --file 20261009230000_agent_crawl.sql+-- Idempotent.+--+-- crawl_missions is the roster: one row per agent sent out to read, holding the+-- topic it reads toward and the seed URLs it starts from. Only the agent's owner+-- writes a row (api/crawl/[action].js action=mission), and the always-on worker+-- (workers/agent-crawler) reads the enabled rows to decide who walks.+--+-- crawl_pages is the corpus: one row per page an agent finished reading, with a+-- short extractive gist and a token count. The full cleaned text lives in object+-- storage under text_key so the table stays small enough for Neon; a page with+-- no storage configured keeps its gist and drops the body.++create table if not exists crawl_missions (+ agent_id uuid primary key references agent_identities(id) on delete cascade,+ topic text not null check (char_length(topic) between 2 and 120),+ seeds text[] not null default '{}',+ enabled boolean not null default true,+ set_by uuid references users(id) on delete set null,+ pages_read bigint not null default 0,+ tokens_read bigint not null default 0,+ last_url text,+ last_at timestamptz,+ created_at timestamptz not null default now(),+ updated_at timestamptz not null default now()+);++create index if not exists crawl_missions_enabled_idx+ on crawl_missions (last_at asc nulls first)+ where enabled = true;++create table if not exists crawl_pages (+ id bigserial primary key,+ agent_id uuid not null references agent_identities(id) on delete cascade,+ url text not null,+ url_hash text not null,+ domain text not null,+ title text,+ gist text,+ tokens integer not null default 0,+ links_out integer not null default 0,+ relevance real,+ from_url text,+ text_key text,+ created_at timestamptz not null default now(),+ unique (agent_id, url_hash)+);++create index if not exists crawl_pages_created_idx on crawl_pages (created_at desc);+create index if not exists crawl_pages_agent_idx on crawl_pages (agent_id, created_at desc);+create index if not exists crawl_pages_domain_idx on crawl_pages (domain);+create index if not exists crawl_pages_url_hash_idx on crawl_pages (url_hash);diff --git a/api/_lib/rate-limit.js b/api/_lib/rate-limit.jsindex 51968a879..ad9d4167f 100644--- a/api/_lib/rate-limit.js+++ b/api/_lib/rate-limit.js@@ -851,6 +851,13 @@ export const limits = {// distinct override gets its own isolated bucket under the rl:api:ip prefix.apiIp: (ip, opts = {}) =>getLimiter('api:ip', { limit: 120, window: '1 m', ...opts }).limit(ip),+ // The Crawl (api/crawl/[action].js). Frames are fetched once per step per tile+ // by every viewer of /crawl, so that bucket stays in per-instance memory rather+ // than spending a Redis command per JPEG; it guards against floods, not spend.+ // Reads are the stats/crawlers/pages/live lane; missions are the owner write.+ crawlFrameIp: (ip) => getLimiter('crawl:frame:ip', { limit: 900, window: '1 m', local: true }).limit(ip),+ crawlReadIp: (ip) => getLimiter('crawl:read:ip', { limit: 240, window: '1 m', local: true }).limit(ip),+ crawlMissionIp: (ip) => getLimiter('crawl:mission:ip', { limit: 30, window: '10 m' }).limit(ip),// Free, unauthenticated 3D Studio (api/mcp-studio.js) abuse protection. Every// studio tool routes through a FREE lane (NVIDIA NIM text→3D, HF Spaces// image→3D) — zero marginal vendor cost — because the studio never names thediff --git a/api/crawl/[action].js b/api/crawl/[action].jsnew file mode 100644index 000000000..e44e5ce66--- /dev/null+++ b/api/crawl/[action].js@@ -0,0 +1,606 @@+// The Crawl: three.ws agents reading the open web in real browsers, watched live.+// Doc: docs/crawl.md. Worker: workers/agent-crawler. Page: /crawl.+//+// Routes:+// GET /api/crawl/stats fleet + corpus totals, top domains (cached 15s)+// GET /api/crawl/crawlers enrolled crawlers with avatar + live state+// GET /api/crawl/live SSE: snapshot, then step/sleep/page events+// GET /api/crawl/frame?agent=<uuid> latest JPEG of that crawler's browser+// GET /api/crawl/pages?agent=&limit=&before=&format=json|jsonl+// the corpus, newest first (jsonl = dataset export)+// GET /api/crawl/mission?agent=<uuid> one agent's mission (public summary)+// PUT /api/crawl/mission owner sends an agent out { agentId, topic, seeds[], enabled }+// DELETE /api/crawl/mission?agent=<uuid> owner calls it home+// GET /api/crawl/mine the signed-in owner's agents + their missions+// GET /api/crawl/roster worker: who should be walking (bearer CRAWL_WORKER_SECRET)+// POST /api/crawl/push worker: one step (+ optional finished page) for one crawler+//+// Storage: Redis carries the live layer (crawl:live:<id> step JSON, crawl:frame:<id>+// base64 JPEG, crawl:active sorted set, crawl:feed recent reads); Postgres carries+// the roster (crawl_missions) and the corpus (crawl_pages); object storage holds+// each page's full cleaned text under crawl/text/.++import { timingSafeEqual } from 'node:crypto';+import { cors, error, json, method, rateLimited, readJson, wrap } from '../_lib/http.js';+import { limits, clientIp } from '../_lib/rate-limit.js';+import { getRedis } from '../_lib/redis.js';+import { sql } from '../_lib/db.js';+import { isUuid } from '../_lib/validate.js';+import { getSessionUser, extractBearer } from '../_lib/auth.js';+import { requireCsrf } from '../_lib/csrf.js';+import { objectStorageConfigured, putObject, publicUrlOrNull, thumbnailUrl } from '../_lib/r2.js';+import { resolveAvatarUrl } from '../_lib/avatars.js';+import {+ CRAWL_LIVE_TTL, CRAWL_ACTIVE_WINDOW_MS, isFrameB64, pageMemory, sanitizePage, sanitizeStep, validateMission,+} from '../_lib/crawl.js';++export const maxDuration = 300;++const WORKER_SECRET = process.env.CRAWL_WORKER_SECRET || '';+const LIVE_KEY = (id) => `crawl:live:${id}`;+const FRAME_KEY = (id) => `crawl:frame:${id}`;+const ACTIVE_KEY = 'crawl:active';+const FEED_KEY = 'crawl:feed';+const STATS_KEY = 'crawl:stats:v1';+const FEED_CAP = 60;+const MEMORY_TTL_MS = 30 * 24 * 3600 * 1000;+const MEMORY_MIN_RELEVANCE = 0.12;++function isWorker(req) {+ const bearer = extractBearer(req);+ if (!bearer || WORKER_SECRET.length < 16) return false;+ const a = Buffer.from(bearer);+ const b = Buffer.from(WORKER_SECRET);+ return a.length === b.length && timingSafeEqual(a, b);+}++// Upstash's REST client parses JSON values on read, so a record can come back as+// a string or an already-parsed object. Accept both.+function parseRecord(v) {+ if (v && typeof v === 'object') return v;+ if (typeof v !== 'string') return null;+ try { return JSON.parse(v); } catch { return null; }+}++function avatarFields(row) {+ const isPublic = row.avatar_visibility === 'public' || row.avatar_visibility === 'unlisted';+ let thumb = null;+ if (isPublic && row.thumbnail_key) {+ try { thumb = thumbnailUrl(row.thumbnail_key); } catch { thumb = null; }+ }+ return { glbRow: isPublic && row.storage_key ? row : null, thumb };+}++async function resolveGlb(row) {+ if (!row) return null;+ try {+ const { url } = await resolveAvatarUrl({+ id: row.avatar_id,+ storage_key: row.storage_key,+ baked_storage_key: row.baked_storage_key,+ appearance_hash: row.appearance_hash,+ appearance: row.appearance,+ visibility: row.avatar_visibility,+ });+ return url;+ } catch {+ return null;+ }+}++async function awakeSet(r) {+ if (!r) return new Map();+ const now = Date.now();+ try {+ const raw = await r.zrange(ACTIVE_KEY, now - CRAWL_ACTIVE_WINDOW_MS, now, { byScore: true, withScores: true });+ const out = new Map();+ for (let i = 0; i + 1 < (raw || []).length; i += 2) out.set(String(raw[i]), Number(raw[i + 1]));+ return out;+ } catch {+ return new Map();+ }+}++// ── worker: roster ──────────────────────────────────────────────────────────+async function handleRoster(req, res) {+ if (!isWorker(req)) return error(res, 401, 'unauthorized', 'crawl worker credential required');+ const url = new URL(req.url, 'http://x');+ const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 24));+ const rows = await sql`+ SELECT m.agent_id, m.topic, m.seeds, m.pages_read, i.name+ FROM crawl_missions m+ JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL+ WHERE m.enabled = true+ ORDER BY m.last_at ASC NULLS FIRST+ LIMIT ${limit}+ `;+ return json(res, 200, {+ crawlers: rows.map((r) => ({+ agentId: r.agent_id, name: r.name, topic: r.topic, seeds: r.seeds || [], pagesRead: Number(r.pages_read) || 0,+ })),+ }, { 'cache-control': 'no-store' });+}++// ── worker: push ────────────────────────────────────────────────────────────+async function storeText(page) {+ if (!page.text || !objectStorageConfigured()) return null;+ const key = `crawl/text/${page.hash.slice(0, 2)}/${page.hash}.txt`;+ try {+ await putObject({ key, body: page.text, contentType: 'text/plain; charset=utf-8', metadata: { url: encodeURI(page.url).slice(0, 1024) } });+ return key;+ } catch (err) {+ console.warn('[crawl] text store failed:', err?.message);+ return null;+ }+}++async function recordPage(agentId, mission, page) {+ const [row] = await sql`+ INSERT INTO crawl_pages (agent_id, url, url_hash, domain, title, gist, tokens, links_out, relevance, from_url)+ VALUES (${agentId}, ${page.url}, ${page.hash}, ${page.domain}, ${page.title}, ${page.gist}, ${page.tokens},+ ${page.linksOut}, ${page.relevance}, ${page.fromUrl})+ ON CONFLICT (agent_id, url_hash) DO NOTHING+ RETURNING id+ `;+ if (!row) return { inserted: false };++ const textKey = await storeText(page);+ const [counts] = await sql`+ UPDATE crawl_missions+ SET pages_read = pages_read + 1, tokens_read = tokens_read + ${page.tokens},+ last_url = ${page.url}, last_at = now(), updated_at = now()+ WHERE agent_id = ${agentId}+ RETURNING pages_read, tokens_read+ `;+ if (textKey) await sql`UPDATE crawl_pages SET text_key = ${textKey} WHERE id = ${row.id}`;++ // The agent keeps what it read, but only when its own owner sent it out: an+ // agent's memory is the owner's to shape.+ const ownerSent = mission.set_by && mission.set_by === mission.user_id;+ if (ownerSent && page.gist && (page.relevance == null || page.relevance >= MEMORY_MIN_RELEVANCE)) {+ const salience = 0.25 + 0.4 * (page.relevance ?? 0.3);+ const expires = new Date(Date.now() + MEMORY_TTL_MS).toISOString();+ await sql`+ INSERT INTO agent_memories (agent_id, type, content, tags, context, salience, tier, expires_at)+ VALUES (${agentId}, 'reference', ${pageMemory(page, mission.topic)}, ${['crawl', page.domain]},+ ${JSON.stringify({ source: 'crawl', url: page.url, page_id: row.id })}::jsonb, ${salience}, 'archival', ${expires})+ `.catch((err) => console.warn('[crawl] memory write failed:', err?.message));+ }+ return { inserted: true, id: row.id, pagesRead: Number(counts?.pages_read) || 0 };+}++async function handlePush(req, res) {+ if (!isWorker(req)) return error(res, 401, 'unauthorized', 'crawl worker credential required');+ let body;+ try {+ body = await readJson(req, 1_200_000);+ } catch {+ return error(res, 400, 'invalid_body', 'request body must be JSON under 1.2 MB');+ }+ const agentId = String(body?.agentId || '');+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agentId must be a uuid');++ const [mission] = await sql`+ SELECT m.topic, m.set_by, m.enabled, m.pages_read, i.name, i.user_id+ FROM crawl_missions m+ JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL+ WHERE m.agent_id = ${agentId}+ LIMIT 1+ `;+ if (!mission || !mission.enabled) return error(res, 410, 'not_enrolled', 'this agent is not out crawling');++ const step = sanitizeStep(body.step);+ if (!step) return error(res, 400, 'invalid_step', 'step is required');+ const page = body.page ? sanitizePage(body.page) : null;+ const frame = isFrameB64(body.frame) ? body.frame : null;++ let pagesRead = Number(mission.pages_read) || 0;+ let recorded = null;+ if (page) {+ recorded = await recordPage(agentId, mission, page);+ if (recorded.inserted) pagesRead = recorded.pagesRead;+ }++ const r = getRedis();+ if (!r) return json(res, 200, { ok: true, live: false, pagesRead });++ const now = Date.now();+ const live = {+ agentId, name: mission.name, topic: mission.topic, ts: now, pagesRead,+ frameTs: frame ? now : undefined, ...step,+ };+ if (!frame) {+ // A step without new pixels keeps pointing at the frame already stored.+ const prev = parseRecord(await r.get(LIVE_KEY(agentId)).catch(() => null));+ if (prev?.frameTs) live.frameTs = prev.frameTs;+ }+ const writes = [+ r.set(LIVE_KEY(agentId), JSON.stringify(live), { ex: CRAWL_LIVE_TTL }),+ r.zadd(ACTIVE_KEY, { score: now, member: agentId }),+ r.zremrangebyscore(ACTIVE_KEY, 0, now - CRAWL_ACTIVE_WINDOW_MS * 2),+ ];+ if (frame) writes.push(r.set(FRAME_KEY(agentId), frame, { ex: CRAWL_LIVE_TTL }));+ if (recorded?.inserted) {+ const entry = { agentId, name: mission.name, url: page.url, title: page.title, domain: page.domain, tokens: page.tokens, gist: page.gist, ts: now };+ writes.push(r.lpush(FEED_KEY, JSON.stringify(entry)).then(() => r.ltrim(FEED_KEY, 0, FEED_CAP - 1)));+ }+ await Promise.all(writes);+ return json(res, 200, { ok: true, live: true, pagesRead, recorded: Boolean(recorded?.inserted) });+}++// ── public: frame ───────────────────────────────────────────────────────────+async function handleFrame(req, res) {+ const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');+ const r = getRedis();+ if (!r) return error(res, 503, 'live_unavailable', 'the live layer is offline');+ const b64 = await r.get(FRAME_KEY(agentId));+ if (typeof b64 !== 'string' || !b64) {+ return error(res, 404, 'asleep', 'this crawler has no live frame right now');+ }+ const buf = Buffer.from(b64, 'base64');+ res.writeHead(200, {+ 'content-type': 'image/jpeg',+ 'content-length': buf.length,+ // The page asks with ?t=<frameTs>, so each frame has its own URL.+ 'cache-control': 'public, max-age=30',+ 'x-content-type-options': 'nosniff',+ });+ res.end(buf);+}++// ── public: live SSE ────────────────────────────────────────────────────────+// One poller per server instance, fanned out to every connected viewer, so Redis+// load is flat in the number of viewers.+const hub = { subs: new Set(), timer: null, crawlers: new Map(), feedTs: 0, feed: [] };+const HUB_POLL_MS = 800;++async function readFleet(r) {+ const awake = await awakeSet(r);+ const ids = [...awake.keys()].slice(0, 120);+ if (!ids.length) return [];+ const raw = await r.mget(...ids.map(LIVE_KEY));+ return (raw || []).map(parseRecord).filter(Boolean);+}++async function readFeed(r, n) {+ const raw = await r.lrange(FEED_KEY, 0, n - 1).catch(() => []);+ return (raw || []).map(parseRecord).filter(Boolean);+}++function broadcast(event, data) {+ const line = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;+ for (const s of hub.subs) s(line);+}++async function hubTick() {+ const r = getRedis();+ if (!r || !hub.subs.size) return;+ try {+ const fleet = await readFleet(r);+ const seen = new Set();+ for (const c of fleet) {+ seen.add(c.agentId);+ const prev = hub.crawlers.get(c.agentId);+ if (!prev || prev.ts !== c.ts) {+ hub.crawlers.set(c.agentId, c);+ broadcast('step', c);+ }+ }+ for (const id of [...hub.crawlers.keys()]) {+ if (!seen.has(id)) {+ hub.crawlers.delete(id);+ broadcast('sleep', { agentId: id });+ }+ }+ const recent = await readFeed(r, 10);+ const fresh = recent.filter((e) => e.ts > hub.feedTs).reverse();+ for (const e of fresh) broadcast('page', e);+ if (recent.length) hub.feedTs = Math.max(hub.feedTs, recent[0].ts);+ } catch (err) {+ console.warn('[crawl] hub tick failed:', err?.message);+ }+}++function hubStart() {+ if (hub.timer) return;+ hub.timer = setInterval(hubTick, HUB_POLL_MS);+ hub.timer.unref?.();+}++function hubStop() {+ if (hub.subs.size || !hub.timer) return;+ clearInterval(hub.timer);+ hub.timer = null;+ hub.crawlers.clear();+}++async function handleLive(req, res) {+ const r = getRedis();+ res.writeHead(200, {+ 'content-type': 'text/event-stream; charset=utf-8',+ 'cache-control': 'no-cache, no-transform',+ connection: 'keep-alive',+ 'x-accel-buffering': 'no',+ });+ res.flushHeaders?.();+ let open = true;+ const write = (line) => {+ if (!open || res.writableEnded) return;+ try { res.write(line); } catch { open = false; }+ };++ if (!r) {+ write(`event: snapshot\ndata: ${JSON.stringify({ crawlers: [], feed: [], live: false })}\n\n`);+ } else {+ const [fleet, feed] = await Promise.all([readFleet(r).catch(() => []), readFeed(r, 30)]);+ for (const c of fleet) hub.crawlers.set(c.agentId, c);+ if (feed.length) hub.feedTs = Math.max(hub.feedTs, feed[0].ts);+ write(`event: snapshot\ndata: ${JSON.stringify({ crawlers: fleet, feed, live: true })}\n\n`);+ }++ hub.subs.add(write);+ hubStart();+ const ping = setInterval(() => write('event: ping\ndata: {}\n\n'), 15_000);+ const end = setTimeout(() => { try { res.end(); } catch { /* closed */ } }, 280_000);+ const close = () => {+ open = false;+ clearInterval(ping);+ clearTimeout(end);+ hub.subs.delete(write);+ hubStop();+ };+ req.on('close', close);+ res.on('close', close);+}++// ── public: stats ───────────────────────────────────────────────────────────+async function handleStats(req, res) {+ const r = getRedis();+ let cached = null;+ if (r) cached = parseRecord(await r.get(STATS_KEY).catch(() => null));+ if (!cached) {+ const [[totals], [enrolled], top] = await Promise.all([+ sql`+ SELECT count(*)::bigint AS pages,+ coalesce(sum(tokens), 0)::bigint AS tokens,+ count(DISTINCT domain)::int AS domains,+ count(DISTINCT url_hash)::bigint AS unique_pages,+ count(*) FILTER (WHERE created_at > now() - interval '24 hours')::bigint AS pages_24h+ FROM crawl_pages+ `,+ sql`SELECT count(*)::int AS n FROM crawl_missions WHERE enabled = true`,+ sql`+ SELECT domain, count(*)::int AS pages+ FROM crawl_pages+ WHERE created_at > now() - interval '7 days'+ GROUP BY domain ORDER BY pages DESC LIMIT 12+ `,+ ]);+ cached = {+ pagesRead: Number(totals.pages) || 0,+ uniquePages: Number(totals.unique_pages) || 0,+ tokens: Number(totals.tokens) || 0,+ domains: totals.domains || 0,+ pages24h: Number(totals.pages_24h) || 0,+ crawlersEnrolled: enrolled.n || 0,+ topDomains: top.map((t) => ({ domain: t.domain, pages: t.pages })),+ };+ if (r) await r.set(STATS_KEY, JSON.stringify(cached), { ex: 15 }).catch(() => {});+ }+ const awake = await awakeSet(r);+ return json(res, 200, { ...cached, crawlersAwake: awake.size, ts: Date.now() }, { 'cache-control': 'public, max-age=10' });+}++// ── public: crawlers ────────────────────────────────────────────────────────+async function handleCrawlers(req, res) {+ const r = getRedis();+ const [rows, awake] = await Promise.all([+ sql`+ SELECT m.agent_id, m.topic, m.pages_read, m.tokens_read, m.last_url, m.last_at, m.enabled,+ i.name, i.description,+ a.id AS avatar_id, a.storage_key, a.baked_storage_key, a.appearance_hash, a.appearance,+ a.thumbnail_key, a.visibility AS avatar_visibility+ FROM crawl_missions m+ JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL+ LEFT JOIN avatars a ON a.id = i.avatar_id AND a.deleted_at IS NULL+ WHERE m.enabled = true+ ORDER BY m.last_at DESC NULLS LAST+ LIMIT 120+ `,+ awakeSet(r),+ ]);+ const crawlers = await Promise.all(rows.map(async (row) => {+ const { glbRow, thumb } = avatarFields(row);+ return {+ agentId: row.agent_id,+ name: row.name,+ description: row.description ? String(row.description).slice(0, 200) : null,+ topic: row.topic,+ pagesRead: Number(row.pages_read) || 0,+ tokensRead: Number(row.tokens_read) || 0,+ lastUrl: row.last_url,+ lastAt: row.last_at,+ awake: awake.has(row.agent_id),+ avatarUrl: await resolveGlb(glbRow),+ thumbnail: thumb,+ };+ }));+ crawlers.sort((a, b) => Number(b.awake) - Number(a.awake) || b.pagesRead - a.pagesRead);+ return json(res, 200, { crawlers }, { 'cache-control': 'public, max-age=10' });+}++// ── public: pages (corpus) ──────────────────────────────────────────────────+async function handlePages(req, res) {+ const p = new URL(req.url, 'http://x').searchParams;+ const agent = p.get('agent');+ if (agent && !isUuid(agent)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');+ const limit = Math.min(500, Math.max(1, Number(p.get('limit')) || 40));+ const before = Number(p.get('before'));+ const beforeId = Number.isSafeInteger(before) && before > 0 ? before : null;+ const rows = await sql`+ SELECT p.id, p.agent_id, i.name AS agent_name, p.url, p.domain, p.title, p.gist, p.tokens,+ p.links_out, p.relevance, p.from_url, p.text_key, p.created_at+ FROM crawl_pages p+ JOIN agent_identities i ON i.id = p.agent_id+ WHERE (${agent}::uuid IS NULL OR p.agent_id = ${agent}::uuid)+ AND (${beforeId}::bigint IS NULL OR p.id < ${beforeId}::bigint)+ ORDER BY p.id DESC+ LIMIT ${limit}+ `;+ const pages = rows.map((row) => ({+ id: Number(row.id),+ agentId: row.agent_id,+ agentName: row.agent_name,+ url: row.url,+ domain: row.domain,+ title: row.title,+ gist: row.gist,+ tokens: row.tokens,+ linksOut: row.links_out,+ relevance: row.relevance,+ fromUrl: row.from_url,+ textUrl: publicUrlOrNull(row.text_key),+ readAt: row.created_at,+ }));+ if (p.get('format') === 'jsonl') {+ res.writeHead(200, {+ 'content-type': 'application/x-ndjson; charset=utf-8',+ 'cache-control': 'public, max-age=30',+ 'content-disposition': 'inline; filename="three-ws-crawl.jsonl"',+ });+ res.end(pages.map((x) => JSON.stringify(x)).join('\n') + (pages.length ? '\n' : ''));+ return;+ }+ const next = pages.length === limit ? pages[pages.length - 1].id : null;+ return json(res, 200, { pages, next }, { 'cache-control': 'public, max-age=10' });+}++// ── mission (public read, owner write) ──────────────────────────────────────+async function ownedAgent(agentId, userId) {+ const [row] = await sql`+ SELECT id, name FROM agent_identities+ WHERE id = ${agentId} AND user_id = ${userId} AND deleted_at IS NULL+ LIMIT 1+ `;+ return row || null;+}++function missionView(row) {+ return {+ agentId: row.agent_id,+ name: row.name,+ topic: row.topic,+ seeds: row.seeds || [],+ enabled: row.enabled,+ pagesRead: Number(row.pages_read) || 0,+ tokensRead: Number(row.tokens_read) || 0,+ lastUrl: row.last_url,+ lastAt: row.last_at,+ };+}++async function handleMission(req, res) {+ if (req.method === 'GET') {+ const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');+ const [row] = await sql`+ SELECT m.*, i.name FROM crawl_missions m+ JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL+ WHERE m.agent_id = ${agentId} LIMIT 1+ `;+ if (!row) return json(res, 200, { mission: null }, { 'cache-control': 'no-store' });+ return json(res, 200, { mission: missionView(row) }, { 'cache-control': 'no-store' });+ }++ const auth = await getSessionUser(req, res);+ const userId = auth?.id;+ if (!userId) return error(res, 401, 'unauthorized', 'sign in to send your agent out');+ if (!(await requireCsrf(req, res, userId))) return;+ const rl = await limits.crawlMissionIp(clientIp(req));+ if (!rl.success) return rateLimited(res, rl);++ if (req.method === 'DELETE') {+ const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');+ if (!(await ownedAgent(agentId, userId))) return error(res, 403, 'forbidden', 'you do not own this agent');+ await sql`UPDATE crawl_missions SET enabled = false, updated_at = now() WHERE agent_id = ${agentId}`;+ const r = getRedis();+ if (r) await Promise.all([r.del(LIVE_KEY(agentId)), r.zrem(ACTIVE_KEY, agentId)]).catch(() => {});+ return json(res, 200, { ok: true, enabled: false });+ }++ let body;+ try { body = await readJson(req, 16_000); } catch { return error(res, 400, 'invalid_body', 'body must be JSON'); }+ const agentId = String(body?.agentId || '');+ if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agentId must be a uuid');+ const agent = await ownedAgent(agentId, userId);+ if (!agent) return error(res, 403, 'forbidden', 'you do not own this agent');+ const m = validateMission(body);+ if (m.error) return error(res, 400, 'invalid_mission', m.error);++ const [row] = await sql`+ INSERT INTO crawl_missions (agent_id, topic, seeds, enabled, set_by)+ VALUES (${agentId}, ${m.topic}, ${m.seeds}, ${m.enabled}, ${userId})+ ON CONFLICT (agent_id) DO UPDATE+ SET topic = excluded.topic, seeds = excluded.seeds, enabled = excluded.enabled,+ set_by = excluded.set_by, updated_at = now()+ RETURNING *+ `;+ return json(res, 200, { mission: missionView({ ...row, name: agent.name }) });+}++async function handleMine(req, res) {+ const auth = await getSessionUser(req, res);+ const userId = auth?.id;+ if (!userId) return error(res, 401, 'unauthorized', 'sign in to see your agents');+ const rows = await sql`+ SELECT i.id, i.name, a.thumbnail_key, a.visibility AS avatar_visibility,+ m.topic, m.seeds, m.enabled, m.pages_read, m.tokens_read, m.last_url, m.last_at+ FROM agent_identities i+ LEFT JOIN avatars a ON a.id = i.avatar_id AND a.deleted_at IS NULL+ LEFT JOIN crawl_missions m ON m.agent_id = i.id+ WHERE i.user_id = ${userId} AND i.deleted_at IS NULL+ ORDER BY m.enabled DESC NULLS LAST, i.created_at DESC+ LIMIT 60+ `;+ return json(res, 200, {+ agents: rows.map((row) => ({+ agentId: row.id,+ name: row.name,+ thumbnail: avatarFields(row).thumb,+ mission: row.topic ? missionView({ ...row, agent_id: row.id }) : null,+ })),+ }, { 'cache-control': 'no-store' });+}++const ROUTES = {+ roster: { methods: ['GET'], fn: handleRoster },+ push: { methods: ['POST'], fn: handlePush },+ frame: { methods: ['GET'], fn: handleFrame },+ live: { methods: ['GET'], fn: handleLive },+ stats: { methods: ['GET'], fn: handleStats },+ crawlers: { methods: ['GET'], fn: handleCrawlers },+ pages: { methods: ['GET'], fn: handlePages },+ mission: { methods: ['GET', 'PUT', 'DELETE'], fn: handleMission },+ mine: { methods: ['GET'], fn: handleMine },+};++export default wrap(async function handleCrawl(req, res) {+ const action = new URL(req.url, 'http://x').searchParams.get('action') || '';+ const route = ROUTES[action];+ if (cors(req, res, { methods: `${route ? route.methods.join(',') : 'GET'},OPTIONS` })) return;+ if (!route) return error(res, 404, 'not_found', 'unknown crawl action');+ if (!method(req, res, route.methods)) return;++ // Worker traffic is authenticated by secret and paced by the worker itself.+ // Frames get a wider bucket (a wall of tiles fetches one per step each).+ if (action === 'frame') {+ const rl = await limits.crawlFrameIp(clientIp(req));+ if (!rl.success) return rateLimited(res, rl);+ } else if (action !== 'push' && action !== 'roster') {+ const rl = await limits.crawlReadIp(clientIp(req));+ if (!rl.success) return rateLimited(res, rl);+ }+ return route.fn(req, res);+});diff --git a/vercel.json b/vercel.jsonindex 0866ad086..14530e47c 100644--- a/vercel.json+++ b/vercel.json@@ -1662,6 +1662,10 @@"src": "/rig-doctor/?","dest": "/rig-doctor.html"},+ {+ "src": "/crawl/?",+ "dest": "/crawl.html"+ },{"src": "/avatar-artifact/?","dest": "/avatar-artifact.html"@@ -3932,6 +3936,42 @@"src": "/api/agent-screen-active","dest": "/api/agent-screen-active.js"},+ {+ "src": "/api/crawl/stats",+ "dest": "/api/crawl/[action]?action=stats"+ },+ {+ "src": "/api/crawl/crawlers",+ "dest": "/api/crawl/[action]?action=crawlers"+ },+ {+ "src": "/api/crawl/live",+ "dest": "/api/crawl/[action]?action=live"+ },+ {+ "src": "/api/crawl/frame",+ "dest": "/api/crawl/[action]?action=frame"+ },+ {+ "src": "/api/crawl/pages",+ "dest": "/api/crawl/[action]?action=pages"+ },+ {+ "src": "/api/crawl/mission",+ "dest": "/api/crawl/[action]?action=mission"+ },+ {+ "src": "/api/crawl/mine",+ "dest": "/api/crawl/[action]?action=mine"+ },+ {+ "src": "/api/crawl/roster",+ "dest": "/api/crawl/[action]?action=roster"+ },+ {+ "src": "/api/crawl/push",+ "dest": "/api/crawl/[action]?action=push"+ },{"src": "/api/agent-task","dest": "/api/agent-task.js"diff --git a/workers/agent-crawler/.dockerignore b/workers/agent-crawler/.dockerignorenew file mode 100644index 000000000..158d407b4--- /dev/null+++ b/workers/agent-crawler/.dockerignore@@ -0,0 +1,6 @@+node_modules+npm-debug.log+README.md+Dockerfile+.dockerignore+cloudbuild.yamldiff --git a/workers/agent-crawler/Dockerfile b/workers/agent-crawler/Dockerfilenew file mode 100644index 000000000..a1ae638f5--- /dev/null+++ b/workers/agent-crawler/Dockerfile@@ -0,0 +1,27 @@+# syntax=docker/dockerfile:1+# agent-crawler: three.ws agents reading the open web in real browsers (/crawl).+#+# The base image ships Chromium plus every system dependency it needs. Its tag+# MUST match the playwright version pinned (exactly) in package.json and the+# lockfile: the image bakes the browser build that playwright release expects,+# so a drifted pair dies on launch with "Executable doesn't exist at+# /ms-playwright/chromium-<rev>". Bump both together.+FROM mcr.microsoft.com/playwright:v1.61.1-noble++WORKDIR /app++COPY package.json package-lock.json ./+RUN npm ci --omit=dev++COPY index.js crawler.js frontier.js extract.js net.js seeds.js ./++ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright+ENV NODE_ENV=production \+ BASE_URL=https://three.ws \+ CRAWLERS=8 \+ JPEG_QUALITY=55++# CRAWL_WORKER_SECRET must be supplied at runtime.+USER pwuser++CMD ["node", "index.js"]diff --git a/workers/agent-crawler/cloudbuild.yaml b/workers/agent-crawler/cloudbuild.yamlnew file mode 100644index 000000000..128b4f21f--- /dev/null+++ b/workers/agent-crawler/cloudbuild.yaml@@ -0,0 +1,80 @@+# Cloud Build to Cloud Run for the crawler fleet behind /crawl (workers/agent-crawler).+#+# Background daemon deployed as a SERVICE: the roster loop runs continuously and+# index.js binds a liveness endpoint on $PORT so the Cloud Run startup probe+# passes. --no-cpu-throttling and --min-instances=1 keep the crawlers walking+# between probes; --max-instances=1 because two instances would both claim the+# same roster and push interleaved steps for the same agent.+#+# Build context is THIS directory:+#+# gcloud builds submit workers/agent-crawler \+# --config workers/agent-crawler/cloudbuild.yaml \+# --region us-central1 --project aerial-vehicle-466722-p5 \+# --substitutions=SHORT_SHA=manual$(date +%s)+#+# The shared secret lives in Secret Manager as `crawl-worker-secret` (three-ws@+# holds secretAccessor). The API reads the same value as CRAWL_WORKER_SECRET;+# both must match or every roster poll answers 401.+#+# Builds run as the pinned three-ws-build service account: the project's default+# compute service account was deleted.+serviceAccount: projects/aerial-vehicle-466722-p5/serviceAccounts/[email protected]++substitutions:+ _REGION: us-central1+ _SERVICE: agent-crawler+ _REPO: workers+ _BASE_URL: https://three.ws+ # Agents walking at once. Each holds one browser context (~300 MB peak with a+ # heavy page), so 8 fits 4Gi with headroom. Raise memory with it.+ _CRAWLERS: "8"++steps:+ - id: build+ name: gcr.io/cloud-builders/docker+ args:+ - build+ - --tag+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:$SHORT_SHA+ - --tag+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:latest+ - --cache-from+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:latest+ - .+ timeout: 1200s++ - id: push+ name: gcr.io/cloud-builders/docker+ args:+ - push+ - --all-tags+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}++ - id: deploy+ name: gcr.io/google.com/cloudsdktool/cloud-sdk:slim+ entrypoint: gcloud+ args:+ - run+ - deploy+ - ${_SERVICE}+ - --image=${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:$SHORT_SHA+ - --region=${_REGION}+ - --platform=managed+ - --no-cpu-throttling+ - --cpu=2+ - --memory=4Gi+ - --min-instances=1+ - --max-instances=1+ - --no-allow-unauthenticated+ - --service-account=three-ws@$PROJECT_ID.iam.gserviceaccount.com+ - --set-env-vars=NODE_ENV=production,BASE_URL=${_BASE_URL},CRAWLERS=${_CRAWLERS}+ - --set-secrets=CRAWL_WORKER_SECRET=crawl-worker-secret:latest++images:+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:$SHORT_SHA+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_SERVICE}:latest++options:+ machineType: E2_HIGHCPU_8+ logging: CLOUD_LOGGING_ONLYdiff --git a/workers/agent-crawler/crawler.js b/workers/agent-crawler/crawler.jsnew file mode 100644index 000000000..9e8990ac9--- /dev/null+++ b/workers/agent-crawler/crawler.js@@ -0,0 +1,375 @@+// One agent out reading: a browser context of its own, a frontier of leads, and+// a loop that reads a page, chooses a link, walks to it on screen, and leaps.+//+// Every phase is pushed to the API as a live step (api/crawl/[action].js+// action=push) so /crawl can draw the agent's avatar walking across the real+// screenshot to the link it really chose. The page it finished reading rides+// along with the "reading" step and lands in the open corpus.++import { readPage, revealLink, visibleLinks, CONSENT_CSS } from './extract.js';+import {+ Frontier, canonical, gistOf, hostOf, relevance, scoreLink, thoughtFor, topicTerms,+} from './frontier.js';+import { USER_AGENT, claimDomain, domainReadyIn, guardHost, robotsCheck } from './net.js';+import { searchSeeds } from './seeds.js';++export const VIEWPORT = { width: 1200, height: 750 };+const STEP_LINKS = 48;+const MIN_TEXT_FOR_CORPUS = 200;+const FRAME_B64_MAX = 690_000;+const RECYCLE_PAGE_EVERY = 40;++export class NotEnrolledError extends Error {}++const sleep = (ms) => new Promise((r) => { setTimeout(r, ms); });++export class Crawler {+ constructor({ mission, browser, api, log, opts }) {+ this.agentId = mission.agentId;+ this.name = mission.name;+ this.topic = mission.topic;+ this.seeds = mission.seeds || [];+ this.terms = topicTerms(mission.topic);+ this.browser = browser;+ this.api = api;+ this.opts = opts;+ this.log = (msg) => log(`[${this.name}] ${msg}`);+ this.frontier = new Frontier();+ this.visited = new Set();+ this.domainCounts = new Map();+ this.seq = 0;+ this.stopped = false;+ this.startedAt = Date.now();+ this.pagesRead = mission.pagesRead || 0;+ this.navigations = 0;+ this.currentUrl = null;+ this.status = 'starting';+ this.lastReseedAt = 0;+ this.done = null;+ }++ summary() {+ return {+ agentId: this.agentId, name: this.name, topic: this.topic, status: this.status,+ url: this.currentUrl, frontier: this.frontier.size, visited: this.visited.size,+ pagesRead: this.pagesRead, upMs: Date.now() - this.startedAt,+ };+ }++ updateMission(mission) {+ if (mission.topic !== this.topic) {+ this.topic = mission.topic;+ this.terms = topicTerms(mission.topic);+ this.frontier = new Frontier();+ this.log(`topic changed to "${mission.topic}", starting fresh`);+ }+ this.seeds = mission.seeds || [];+ }++ start() {+ this.done = this.run().catch((err) => {+ if (!(err instanceof NotEnrolledError)) this.log(`stopped on error: ${err?.message || err}`);+ }).finally(() => this.close());+ return this.done;+ }++ async stop() {+ this.stopped = true;+ await this.close();+ await this.done?.catch(() => {});+ }++ async close() {+ const ctx = this.context;+ this.context = null;+ this.page = null;+ if (ctx) await ctx.close().catch(() => {});+ }++ async openContext() {+ this.context = await this.browser.newContext({+ viewport: VIEWPORT,+ deviceScaleFactor: 1,+ userAgent: USER_AGENT,+ locale: 'en-US',+ acceptDownloads: false,+ javaScriptEnabled: true,+ serviceWorkers: 'block',+ });+ await this.context.addInitScript((css) => {+ const add = () => {+ if (document.getElementById('tws-crawl-consent')) return;+ const s = document.createElement('style');+ s.id = 'tws-crawl-consent';+ s.textContent = css;+ (document.head || document.documentElement).appendChild(s);+ };+ if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', add, { once: true });+ else add();+ }, CONSENT_CSS);+ // Every request is fenced: private hosts are refused and media streams+ // (which never show in a still frame) are skipped.+ await this.context.route('**/*', async (route) => {+ const req = route.request();+ if (req.resourceType() === 'media') return route.abort('blockedbyclient');+ let host = '';+ try { host = new URL(req.url()).hostname; } catch { return route.abort('blockedbyclient'); }+ if (req.url().startsWith('data:') || req.url().startsWith('blob:')) return route.continue();+ if (!(await guardHost(host))) return route.abort('blockedbyclient');+ return route.continue();+ });+ this.context.on('page', (p) => {+ if (this.page && p !== this.page) p.close().catch(() => {});+ });+ await this.newPage();+ }++ async newPage() {+ if (this.page) await this.page.close().catch(() => {});+ this.page = await this.context.newPage();+ this.page.on('dialog', (d) => d.dismiss().catch(() => {}));+ this.navigations = 0;+ }++ async run() {+ await this.openContext();+ await this.loadHistory();+ await this.seed();++ // A walked-to link is visited straight away (the avatar is standing on+ // it); otherwise the frontier picks the next lead.+ let following = null;+ while (!this.stopped) {+ const next = following || this.pickNext();+ following = null;+ if (!next) {+ await this.reseedOrRest();+ continue;+ }+ following = await this.visit(next);+ }+ }++ // Pages this agent already read survive restarts: the corpus is the memory.+ async loadHistory() {+ try {+ const pages = await this.api.recentPages(this.agentId, 500);+ for (const p of pages) {+ const key = canonical(p.url);+ if (key) this.visited.add(key);+ const h = hostOf(p.url);+ this.domainCounts.set(h, Math.min(10, (this.domainCounts.get(h) || 0) + 1));+ }+ } catch (err) {+ this.log(`history unavailable (${err?.message || err}); starting with a clean slate`);+ }+ }++ async seed() {+ for (const url of this.seeds) this.frontier.add(url, 50, null, 'start page');+ if (!this.frontier.size) await this.searchForLeads();+ }++ async searchForLeads() {+ this.lastReseedAt = Date.now();+ const { urls, source } = await searchSeeds(this.topic, this.log);+ urls.forEach((u, i) => this.frontier.add(u, 20 - i * 0.5, null, ''));+ if (urls.length) this.log(`${urls.length} leads from ${source}`);+ return urls.length;+ }++ async reseedOrRest() {+ const sinceReseed = Date.now() - this.lastReseedAt;+ if (sinceReseed > 120_000) {+ for (const url of this.seeds) if (!this.visited.has(canonical(url))) this.frontier.add(url, 50);+ if (this.frontier.size || (await this.searchForLeads())) return;+ }+ this.status = 'resting';+ await this.pushStep({+ url: this.currentUrl, title: '', status: 'resting', links: [], target: null,+ thought: thoughtFor('resting', { reason: `Out of fresh leads on ${this.topic}. Searching again shortly.` }),+ });+ await this.idle(60_000);+ }++ async idle(ms) {+ const until = Date.now() + ms;+ while (!this.stopped && Date.now() < until) await sleep(Math.min(1000, until - Date.now()));+ }++ pickNext() {+ const accept = (item) => {+ if (this.visited.has(item.url)) return false;+ return domainReadyIn(hostOf(item.url)) <= 2500;+ };+ let item = this.frontier.pop(accept);+ if (!item && this.frontier.size) {+ // Everything left is on a cooling domain: take the best one; visit()+ // waits out its politeness gap before navigating.+ item = this.frontier.pop((i) => !this.visited.has(i.url));+ }+ return item;+ }++ async pushStep(step, extra = {}) {+ this.seq += 1;+ return this.api.push({ agentId: this.agentId, step: { ...step, seq: this.seq }, ...extra });+ }++ async screenshot() {+ let quality = this.opts.jpegQuality;+ for (let attempt = 0; attempt < 3; attempt++) {+ const buf = await this.page.screenshot({ type: 'jpeg', quality, timeout: 8000, animations: 'disabled', caret: 'hide' });+ const b64 = buf.toString('base64');+ if (b64.length <= FRAME_B64_MAX) return b64;+ quality = Math.max(25, quality - 15);+ }+ return null;+ }++ async blocked(host, reason) {+ this.status = 'blocked';+ await this.pushStep({+ url: this.currentUrl, title: '', status: 'blocked', links: [], target: null,+ thought: thoughtFor('blocked', { host, reason }),+ }).catch((err) => { if (err instanceof NotEnrolledError) throw err; });+ }++ // Read one page, then decide where to go. Returns the frontier entry to visit+ // next when the agent chose an on-screen link (it already walked there), or+ // null to let the frontier decide.+ async visit(target) {+ const url = canonical(target.url);+ if (!url) return null;+ const host = hostOf(url);+ this.visited.add(url);++ const robots = await robotsCheck(url).catch(() => ({ allowed: false, delayMs: 0 }));+ if (!robots.allowed) {+ await this.blocked(host, 'asks crawlers to stay out of that page');+ return null;+ }+ const wait = domainReadyIn(host);+ if (wait > 0) await sleep(wait);+ claimDomain(host, Math.max(this.opts.domainGapMs, robots.delayMs));++ if (this.navigations >= RECYCLE_PAGE_EVERY) await this.newPage();+ this.navigations += 1;++ let resp;+ try {+ resp = await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.opts.navTimeoutMs });+ } catch (err) {+ const msg = String(err?.message || err);+ await this.blocked(host, /ERR_BLOCKED_BY_CLIENT/.test(msg) ? 'is not a public address' : 'did not answer in time');+ if (/Target closed|crashed/i.test(msg)) await this.newPage();+ return null;+ }+ if (!resp || resp.status() >= 400) {+ await this.blocked(host, `answered ${resp ? resp.status() : 'nothing'}`);+ return null;+ }+ const ctype = resp.headers()['content-type'] || '';+ if (ctype && !/html|xhtml/i.test(ctype)) {+ await this.blocked(host, 'sent a file, not a page');+ return null;+ }+ // Let the page paint its above-the-fold content before the frame is taken.+ await this.page.waitForLoadState('load', { timeout: 6000 }).catch(() => {});++ const finalUrl = canonical(this.page.url()) || url;+ this.visited.add(finalUrl);+ const finalHost = hostOf(finalUrl);+ this.domainCounts.set(finalHost, (this.domainCounts.get(finalHost) || 0) + 1);+ this.currentUrl = finalUrl;++ const data = await this.page.evaluate(readPage, 600);+ const rel = relevance(this.terms, data.title, data.text);+ const gist = gistOf(data.text, this.terms);++ // Score every link on the page into the frontier. Links found on a+ // relevant page inherit some of its relevance (topical locality).+ const ctx = { terms: this.terms, fromHost: finalHost, domainCounts: this.domainCounts, visited: this.visited, rand: Math.random };+ let best = null;+ for (const link of data.links) {+ const s = scoreLink(link, ctx);+ if (!Number.isFinite(s)) continue;+ const score = s + rel * 1.5;+ this.frontier.add(link.href, score, finalUrl, link.text);+ if (!best || score > best.score) best = { ...link, score };+ }+ this.frontier.decay();++ const frame = await this.screenshot();+ const shown = data.links.filter((l) => l.visible).slice(0, STEP_LINKS).map(({ x, y, w, h, text }) => ({ x, y, w, h, t: text }));+ const page = data.text.length >= MIN_TEXT_FOR_CORPUS ? {+ url: finalUrl, title: data.title, text: data.text, gist, linksOut: data.links.length, relevance: rel, fromUrl: target.from,+ } : null;++ this.status = 'reading';+ const res = await this.pushStep({+ url: finalUrl, title: data.title, status: 'reading', links: shown, target: null, scrollY: data.scrollY,+ thought: thoughtFor('reading', { title: data.title || finalHost, domain: finalHost, topic: this.topic, relevance: rel }),+ }, { frame, page });+ if (res?.recorded) this.pagesRead = res.pagesRead || this.pagesRead + 1;++ // Read: dwell on the page roughly in proportion to how much there is.+ await this.idle(this.opts.readMs + Math.min(3000, data.text.length / 20));+ if (this.stopped) return null;++ const frontierBest = [...this.frontier.items.values()]+ .filter((i) => !this.visited.has(i.url) && i.url !== canonical(best?.href))+ .reduce((m, i) => (!m || i.score > m.score ? i : m), null);+ if (best && (!frontierBest || best.score >= frontierBest.score - 0.5)) {+ const walked = await this.walkTo(best, finalUrl, finalHost);+ if (walked) return walked;+ }++ // Leap: the best lead is not on this screen.+ const lead = frontierBest;+ if (lead) {+ this.status = 'leaping';+ await this.pushStep({+ url: finalUrl, title: data.title, status: 'leaping', links: [], target: null,+ thought: thoughtFor('leaping', { text: lead.text, host: hostOf(lead.url) }),+ });+ await this.idle(this.opts.leapMs);+ }+ return null;+ }++ // Bring the chosen link on screen, push the walking step with its rect as the+ // target, give the avatar time to walk there, then hop.+ async walkTo(link, pageUrl, pageHost) {+ const ok = await this.page.evaluate(revealLink, link.i).catch(() => false);+ if (!ok) return null;+ const measured = await this.page.evaluate(visibleLinks, 400).catch(() => null);+ if (!measured) return null;+ let list = measured.links;+ let idx = list.findIndex((l) => l.i === link.i);+ if (idx < 0) return null;+ if (list.length > STEP_LINKS) {+ const tgt = list[idx];+ list = [tgt, ...list.filter((l) => l !== tgt).slice(0, STEP_LINKS - 1)];+ idx = 0;+ }+ const frame = link.visible ? null : await this.screenshot();+ const host = hostOf(link.href);+ this.status = 'walking';+ await this.pushStep({+ url: pageUrl, title: '', status: 'walking', scrollY: measured.scrollY,+ links: list.map(({ x, y, w, h, t }) => ({ x, y, w, h, t })), target: idx, nextUrl: link.href,+ thought: thoughtFor('walking', { text: link.text, host, offsite: host !== pageHost }),+ }, frame ? { frame } : {});+ await this.idle(this.opts.walkMs);+ if (this.stopped) return null;+ await this.pushStep({+ url: pageUrl, title: '', status: 'leaping', scrollY: measured.scrollY,+ links: list.map(({ x, y, w, h, t }) => ({ x, y, w, h, t })), target: idx, nextUrl: link.href,+ thought: thoughtFor('walking', { text: link.text, host, offsite: host !== pageHost }),+ });+ await this.idle(this.opts.leapMs);+ this.frontier.drop(link.href);+ return { url: link.href, score: link.score, from: pageUrl, text: link.text };+ }+}diff --git a/workers/agent-crawler/extract.js b/workers/agent-crawler/extract.jsnew file mode 100644index 000000000..e9be4ed8c--- /dev/null+++ b/workers/agent-crawler/extract.js@@ -0,0 +1,122 @@+// What the crawler reads off a live page. Every function exported here runs+// INSIDE the page via page.evaluate, so each one must be self-contained: no+// closures over module scope, no imports.++// Read the page: title, cleaned main text, and every followable anchor with its+// on-screen rectangle. Anchors get a data-tws-crawl index so the crawler can+// scroll a chosen one into view and re-measure it without a fragile selector.+export function readPage(maxLinks) {+ const vw = window.innerWidth;+ const vh = window.innerHeight;++ const pickRoot = () => {+ const candidates = [...document.querySelectorAll('main, article, [role="main"], #content, #main, .content, .post, .article')];+ let best = null;+ let bestLen = 0;+ for (const el of candidates) {+ const len = (el.innerText || '').length;+ if (len > bestLen) { best = el; bestLen = len; }+ }+ return best && bestLen > 600 ? best : document.body;+ };++ const cleanText = (root) => {+ if (!root) return '';+ const clone = root.cloneNode(true);+ clone.querySelectorAll('script, style, noscript, svg, canvas, iframe, form, nav, footer, header, aside, button, select, [aria-hidden="true"], [hidden]').forEach((n) => n.remove());+ // innerText on a detached clone ignores layout, so fall back to textContent+ // with block elements turned into line breaks.+ clone.querySelectorAll('p, div, li, h1, h2, h3, h4, h5, h6, tr, br, section, article, blockquote, pre').forEach((n) => n.append('\n'));+ return (clone.textContent || '')+ .split('\n')+ .map((l) => l.replace(/\s+/g, ' ').trim())+ .filter((l) => l.length > 1)+ .join('\n');+ };++ const isShown = (el) => {+ const cs = getComputedStyle(el);+ return cs.visibility !== 'hidden' && cs.display !== 'none' && Number(cs.opacity) > 0.05;+ };++ const anchors = [...document.querySelectorAll('a[href]')];+ const links = [];+ for (let i = 0; i < anchors.length && links.length < maxLinks; i++) {+ const a = anchors[i];+ const href = a.href;+ if (!href || !/^https?:/i.test(href)) continue;+ a.setAttribute('data-tws-crawl', String(i));+ const r = a.getBoundingClientRect();+ const sized = r.width >= 6 && r.height >= 6;+ const onScreen = sized && r.bottom > 0 && r.right > 0 && r.top < vh && r.left < vw;+ links.push({+ i,+ href,+ text: (a.innerText || a.getAttribute('aria-label') || a.title || '').replace(/\s+/g, ' ').trim().slice(0, 140),+ visible: onScreen && isShown(a),+ x: r.left / vw,+ y: r.top / vh,+ w: r.width / vw,+ h: r.height / vh,+ });+ }++ const docH = Math.max(document.documentElement.scrollHeight, document.body?.scrollHeight || 0, vh);+ return {+ title: (document.title || document.querySelector('h1')?.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 200),+ lang: document.documentElement.lang || '',+ text: cleanText(pickRoot()).slice(0, 120000),+ links,+ scrollY: docH > vh ? window.scrollY / (docH - vh) : 0,+ };+}++// Bring one anchor to the middle of the screen. Returns false when the anchor+// vanished (single-page apps re-render).+export function revealLink(index) {+ const a = document.querySelector(`a[data-tws-crawl="${index}"]`);+ if (!a) return false;+ a.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });+ return true;+}++// Re-measure only what is on screen right now, in the shape the live step+// carries: normalized rects clipped to the viewport.+export function visibleLinks(maxLinks) {+ const vw = window.innerWidth;+ const vh = window.innerHeight;+ const out = [];+ for (const a of document.querySelectorAll('a[data-tws-crawl]')) {+ if (out.length >= maxLinks) break;+ const r = a.getBoundingClientRect();+ if (r.width < 6 || r.height < 6 || r.bottom <= 0 || r.right <= 0 || r.top >= vh || r.left >= vw) continue;+ const cs = getComputedStyle(a);+ if (cs.visibility === 'hidden' || cs.display === 'none' || Number(cs.opacity) <= 0.05) continue;+ const x = Math.max(0, r.left);+ const y = Math.max(0, r.top);+ out.push({+ i: Number(a.getAttribute('data-tws-crawl')),+ href: a.href,+ t: (a.innerText || a.getAttribute('aria-label') || '').replace(/\s+/g, ' ').trim().slice(0, 80),+ x: x / vw,+ y: y / vh,+ w: (Math.min(vw, r.right) - x) / vw,+ h: (Math.min(vh, r.bottom) - y) / vh,+ });+ }+ const docH = Math.max(document.documentElement.scrollHeight, document.body?.scrollHeight || 0, vh);+ return { links: out, scrollY: docH > vh ? window.scrollY / (docH - vh) : 0 };+}++// Consent walls cover the very page viewers came to watch. Hide the common+// vendors' overlays (never click them: the crawler accepts nothing on anyone's+// behalf). Injected as an init script on every document.+export const CONSENT_CSS = [+ '#onetrust-banner-sdk', '#onetrust-consent-sdk', '.onetrust-pc-dark-filter', '#CybotCookiebotDialog',+ '#CybotCookiebotDialogBodyUnderlay', '.fc-consent-root', '.qc-cmp2-container', '#qc-cmp2-container',+ '#truste-consent-track', '.truste_box_overlay', '.cc-window', '.cc-banner', '#cookie-banner', '#cookieBanner',+ '#cookie-notice', '.cookie-notice', '#cookie-consent', '.cookie-consent', '#gdpr-banner', '.gdpr-banner',+ '[id^="sp_message_container"]', '.sp_veil', '#usercentrics-root', '#didomi-host', '.didomi-popup-backdrop',+ '#iubenda-cs-banner', '.osano-cm-window', '#hs-eu-cookie-confirmation',+].join(',') + '{display:none !important;visibility:hidden !important}'+ + 'html,body{overflow:auto !important}';diff --git a/workers/agent-crawler/frontier.js b/workers/agent-crawler/frontier.jsnew file mode 100644index 000000000..b751fd92b--- /dev/null+++ b/workers/agent-crawler/frontier.js@@ -0,0 +1,241 @@+// Link choice for one crawler: which page does the agent walk to next?+//+// Pure on purpose (no browser, no network) so the root test suite can pin the+// behavior: tests/agent-crawler-frontier.test.js. The crawler hands every link it+// measured on a page to scoreLink(), keeps the best candidates in a Frontier, and+// walks to the winner when it is on screen or leaps to it when it is not.++const STOPWORDS = new Set(+ ('a an and are as at be but by for from has have how in into is it its of on or that the their this to was were what '+ + 'when where which who why will with about after all also any can could did does each more most new not now our out '+ + 'over so some such than them then there these they through up use used using very was way we you your').split(' '),+);++// Links a reading agent should never follow: account walls, carts, share+// intents, and anything that downloads instead of rendering.+const SKIP_PATH = /\/(login|log-in|signin|sign-in|signup|sign-up|register|logout|log-out|cart|checkout|basket|account|my-account|subscribe|password|auth|oauth|wp-admin|wp-login)(\/|$|\?|\.)/i;+const SKIP_HOST = /(^|\.)(facebook\.com|instagram\.com|tiktok\.com|x\.com|twitter\.com|t\.co|linkedin\.com|pinterest\.com|reddit\.com\/submit|accounts\.google\.com|doubleclick\.net|googleadservices\.com)$/i;+const SKIP_EXT = /\.(pdf|zip|gz|tgz|rar|7z|exe|dmg|msi|iso|apk|deb|rpm|jpg|jpeg|png|gif|webp|avif|svg|ico|bmp|tiff?|mp3|mp4|m4a|wav|ogg|webm|mov|avi|mkv|woff2?|ttf|otf|css|js|json|xml|rss|atom|csv|xlsx?|docx?|pptx?)$/i;+const SHARE = /(sharer|share\?|intent\/tweet|shareArticle|mailto:|javascript:|tel:|sms:|whatsapp:)/i;+const BOILERPLATE = /^(privacy|privacy policy|terms|terms of (use|service)|cookies?|cookie (policy|settings)|contact( us)?|careers|jobs|advertise|sitemap|accessibility|imprint|legal|help|faq|log ?in|sign ?(in|up)|subscribe|donate|skip to (main )?content|menu|search|home)$/i;++export const MAX_FRONTIER = 400;+export const MAX_PAGES_PER_DOMAIN = 30;++// Lowercase content words with a light suffix strip, so "crawlers" in a topic+// matches "crawler" in an anchor and "learning" matches "learn".+export function stem(word) {+ let w = word.toLowerCase();+ if (w.length > 5 && w.endsWith('ing')) w = w.slice(0, -3);+ else if (w.length > 4 && w.endsWith('ies')) w = `${w.slice(0, -3)}y`;+ else if (w.length > 4 && w.endsWith('ed')) w = w.slice(0, -2);+ else if (w.length > 3 && w.endsWith('s') && !w.endsWith('ss')) w = w.slice(0, -1);+ return w;+}++export function tokenize(text) {+ return String(text || '')+ .toLowerCase()+ .split(/[^\p{L}\p{N}]+/u)+ .filter((w) => w.length >= 3 && !STOPWORDS.has(w))+ .map(stem);+}++export function topicTerms(topic) {+ return [...new Set(tokenize(topic))].slice(0, 16);+}++export function hostOf(url) {+ try { return new URL(url).hostname.toLowerCase().replace(/^www\./, ''); } catch { return ''; }+}++// Strip the fragment and tracking tags so the same article reached from two+// pages counts as one visit. Mirrors normalizeUrl in api/_lib/crawl.js closely+// enough for dedupe; the API re-normalizes authoritatively.+export function canonical(url) {+ let u;+ try { u = new URL(url); } catch { return null; }+ if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;+ if (u.username || u.password) return null;+ u.hash = '';+ for (const k of [...u.searchParams.keys()]) {+ if (/^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_[a-z]+|ref|ref_src|igshid|si)$/i.test(k)) u.searchParams.delete(k);+ }+ u.searchParams.sort();+ if (u.pathname.length > 1 && u.pathname.endsWith('/')) u.pathname = u.pathname.replace(/\/+$/, '');+ return u.href;+}++export function isCrawlable(url) {+ if (!url || url.length > 400 || SHARE.test(url)) return false;+ let u;+ try { u = new URL(url); } catch { return false; }+ if (u.protocol !== 'https:' && u.protocol !== 'http:') return false;+ if (SKIP_HOST.test(u.hostname) || SKIP_HOST.test(`${u.hostname}${u.pathname}`)) return false;+ if (SKIP_PATH.test(u.pathname) || SKIP_EXT.test(u.pathname)) return false;+ return true;+}++// Fraction of topic terms present in a token list, 0..1.+export function coverage(terms, tokens) {+ if (!terms.length) return 0;+ const set = tokens instanceof Set ? tokens : new Set(tokens);+ let hit = 0;+ for (const t of terms) if (set.has(t)) hit += 1;+ return hit / terms.length;+}++// How close is a page to the topic? Title hits count double, and body density+// (term mentions per thousand words, saturating) separates a passing mention+// from a page that is about the thing.+export function relevance(terms, title, text) {+ if (!terms.length) return 0;+ const titleCov = coverage(terms, tokenize(title));+ const words = tokenize(String(text || '').slice(0, 40_000));+ if (!words.length) return Math.min(1, titleCov * 0.6);+ const termSet = new Set(terms);+ let mentions = 0;+ for (const w of words) if (termSet.has(w)) mentions += 1;+ const density = Math.min(1, (mentions / words.length) * 1000 / 25);+ const bodyCov = coverage(terms, words);+ return Math.round(Math.min(1, titleCov * 0.4 + bodyCov * 0.35 + density * 0.25) * 1000) / 1000;+}++// Score one link. ctx: { terms, fromHost, domainCounts: Map, visited: Set, rand }.+// Returns -Infinity for links that must never be followed.+export function scoreLink(link, ctx) {+ const url = canonical(link.href);+ if (!url || !isCrawlable(url) || ctx.visited.has(url)) return -Infinity;+ const host = hostOf(url);+ const seen = ctx.domainCounts.get(host) || 0;+ if (seen >= MAX_PAGES_PER_DOMAIN) return -Infinity;++ const text = String(link.text || '').trim();+ if (BOILERPLATE.test(text)) return -Infinity;++ const textTokens = tokenize(text);+ let pathTokens = [];+ try { pathTokens = tokenize(decodeURIComponent(new URL(url).pathname.replace(/[-_/.]+/g, ' '))); } catch { pathTokens = []; }++ const textHits = ctx.terms.filter((t) => textTokens.includes(t)).length;+ const pathHits = ctx.terms.filter((t) => pathTokens.includes(t)).length;+ let score = textHits * 3 + pathHits * 1.5;++ // Descriptive anchors beat "click here" and bare icons.+ if (text.length >= 12 && text.length <= 120) score += 0.6;+ else if (!text) score -= 0.8;++ const sameHost = host === ctx.fromHost;+ if (sameHost) score += 0.4;+ else if (seen === 0 && (textHits || pathHits)) score += 1.2; // a relevant new site widens the corpus+ else if (seen === 0) score += 0.3;+ score -= seen * 0.08; // wear off a domain gradually++ // Prefer article-shaped paths over hubs and query soups.+ const depth = pathTokens.length;+ if (depth >= 2 && depth <= 10) score += 0.3;+ if (url.includes('?') && url.split('&').length > 3) score -= 0.6;++ if (link.visible) score += 0.25; // the agent can walk to it on this screen+ return score + (ctx.rand ? ctx.rand() * 0.35 : 0);+}++// A bounded best-first queue of URLs, deduped by canonical URL.+export class Frontier {+ constructor(max = MAX_FRONTIER) {+ this.max = max;+ this.items = new Map(); // url -> { url, score, from, text }+ }++ get size() { return this.items.size; }++ add(url, score, from = null, text = '') {+ const key = canonical(url);+ if (!key || !Number.isFinite(score)) return false;+ const prev = this.items.get(key);+ if (prev && prev.score >= score) return false;+ this.items.set(key, { url: key, score, from, text });+ if (this.items.size > this.max) this.trim();+ return true;+ }++ trim() {+ const sorted = [...this.items.values()].sort((a, b) => b.score - a.score).slice(0, this.max);+ this.items = new Map(sorted.map((i) => [i.url, i]));+ }++ // Highest-scoring entry that passes the filter (e.g. not visited, not cooling+ // down). Entries that fail are left in place for a later turn.+ pop(accept = () => true) {+ let best = null;+ for (const item of this.items.values()) {+ if (!accept(item)) continue;+ if (!best || item.score > best.score) best = item;+ }+ if (best) this.items.delete(best.url);+ return best;+ }++ drop(url) { this.items.delete(canonical(url)); }++ // Decay everything a little each turn so stale hopes from fifty pages ago+ // lose to fresh leads from the page the agent is actually standing on.+ decay(factor = 0.97) {+ for (const item of this.items.values()) item.score *= factor;+ }+}++// Extractive gist: the first few real sentences, preferring ones that mention+// the topic. No model call, so it is free, deterministic, and never invents.+export function gistOf(text, terms, max = 480) {+ // Lines are block boundaries (readPage puts one per paragraph), so a menu+ // crumb on its own line never glues onto the first real sentence.+ const sentences = String(text || '')+ .split(/\n+/)+ .flatMap((line) => line.replace(/\s+/g, ' ').split(/(?<=[.!?])\s+(?=[A-Z0-9"“(])/))+ .map((s) => s.trim())+ .filter((s) => s.length >= 40 && s.length <= 400 && /[a-z]/.test(s) && (s.match(/\s/g) || []).length >= 5)+ .slice(0, 40);+ if (!sentences.length) return '';+ const termSet = new Set(terms);+ const ranked = sentences.map((s, i) => {+ const hits = tokenize(s).filter((t) => termSet.has(t)).length;+ return { s, i, rank: hits * 2 - i * 0.15 };+ });+ const picked = ranked.sort((a, b) => b.rank - a.rank).slice(0, 3).sort((a, b) => a.i - b.i);+ let out = '';+ for (const { s } of picked) {+ if ((out ? out.length + 1 : 0) + s.length > max) break;+ out = out ? `${out} ${s}` : s;+ }+ return out || sentences[0].slice(0, max);+}++// What the agent "says" on its card. Deterministic phrasing from what it is+// really doing; no model is asked to narrate.+export function thoughtFor(kind, d = {}) {+ const quote = (s, n = 70) => {+ const t = String(s || '').replace(/\s+/g, ' ').trim();+ return t.length > n ? `${t.slice(0, n - 1)}…` : t;+ };+ switch (kind) {+ case 'reading':+ return d.relevance >= 0.35+ ? `Reading "${quote(d.title)}" on ${d.domain}. This is squarely about ${d.topic}.`+ : d.relevance >= 0.12+ ? `Reading "${quote(d.title)}" on ${d.domain}. Some of this touches ${d.topic}.`+ : `Passing through ${d.domain}. Not much about ${d.topic} here.`;+ case 'walking':+ return d.text+ ? `Walking to "${quote(d.text, 60)}"${d.offsite ? `, off to ${d.host}` : ''}.`+ : `Walking to a link${d.offsite ? ` on ${d.host}` : ''}.`;+ case 'leaping':+ return d.text ? `Leaping to "${quote(d.text, 60)}" from my list.` : `Leaping to ${d.host} from my list.`;+ case 'blocked':+ return `${d.host} ${d.reason || 'turned me away'}. Finding another way.`;+ case 'resting':+ return d.reason || 'Out of leads. Looking for new start pages.';+ default:+ return '';+ }+}diff --git a/workers/agent-crawler/index.js b/workers/agent-crawler/index.jsnew file mode 100644index 000000000..0ec13c760--- /dev/null+++ b/workers/agent-crawler/index.js@@ -0,0 +1,204 @@+// agent-crawler: three.ws agents reading the open web in real browsers.+//+// The worker behind /crawl. It polls the roster of agents whose owners sent them+// out (GET /api/crawl/roster), keeps up to CRAWLERS of them walking at once (one+// shared Chromium, one isolated context per agent), and pushes every step and+// every finished page back through POST /api/crawl/push. The API fans steps out+// to viewers live and files pages into the open corpus. Doc: docs/crawl.md.+//+// When more agents are enrolled than there are slots, each walks a shift of+// SHIFT_MS and then yields to the agent that has waited longest (the roster is+// ordered by last read, oldest first), so every enrolled agent gets its turn.+//+// Required env:+// CRAWL_WORKER_SECRET shared secret; must match the API's value.+// Optional env:+// BASE_URL=https://three.ws CRAWLERS=8 SHIFT_MS=1200000 ROSTER_MS=30000+// READ_MS=3500 WALK_MS=3200 LEAP_MS=900 DOMAIN_GAP_MS=4000 NAV_TIMEOUT_MS=20000+// JPEG_QUALITY=55+// PORT when set, bind the liveness endpoint (Cloud Run sets it).++import http from 'node:http';+import { chromium } from 'playwright';+import { Crawler, NotEnrolledError } from './crawler.js';++const BASE_URL = (process.env.BASE_URL || 'https://three.ws').replace(/\/$/, '');+const SECRET = process.env.CRAWL_WORKER_SECRET || '';+const CRAWLERS = Math.max(1, Number(process.env.CRAWLERS || 8));+const SHIFT_MS = Number(process.env.SHIFT_MS || 20 * 60_000);+const ROSTER_MS = Number(process.env.ROSTER_MS || 30_000);+const OPTS = {+ readMs: Number(process.env.READ_MS || 3500),+ walkMs: Number(process.env.WALK_MS || 3200),+ leapMs: Number(process.env.LEAP_MS || 900),+ domainGapMs: Number(process.env.DOMAIN_GAP_MS || 4000),+ navTimeoutMs: Number(process.env.NAV_TIMEOUT_MS || 20_000),+ jpegQuality: Number(process.env.JPEG_QUALITY || 55),+};++const log = (msg) => console.log(`${new Date().toISOString()} ${msg}`);++if (SECRET.length < 16) {+ console.error('CRAWL_WORKER_SECRET is missing or shorter than 16 characters; the API would refuse every call.');+ process.exit(1);+}++// ── API client ──────────────────────────────────────────────────────────────+const api = {+ async roster(limit) {+ const res = await fetch(`${BASE_URL}/api/crawl/roster?limit=${limit}`, {+ headers: { authorization: `Bearer ${SECRET}` },+ signal: AbortSignal.timeout(15_000),+ });+ if (!res.ok) throw new Error(`roster ${res.status}`);+ return (await res.json()).crawlers || [];+ },++ async recentPages(agentId, limit) {+ const res = await fetch(`${BASE_URL}/api/crawl/pages?agent=${agentId}&limit=${limit}`, {+ signal: AbortSignal.timeout(15_000),+ });+ if (!res.ok) throw new Error(`pages ${res.status}`);+ return (await res.json()).pages || [];+ },++ // One retry on a network blip or 5xx; a 410 means the owner called the agent+ // home, which ends that crawler. Anything else is logged and the walk goes on.+ async push(body) {+ const payload = JSON.stringify(body);+ for (let attempt = 0; attempt < 2; attempt++) {+ try {+ const res = await fetch(`${BASE_URL}/api/crawl/push`, {+ method: 'POST',+ headers: { authorization: `Bearer ${SECRET}`, 'content-type': 'application/json' },+ body: payload,+ signal: AbortSignal.timeout(20_000),+ });+ if (res.status === 410) throw new NotEnrolledError('mission ended');+ if (res.ok) return await res.json();+ if (res.status < 500) {+ log(`push rejected ${res.status}: ${(await res.text()).slice(0, 200)}`);+ return null;+ }+ } catch (err) {+ if (err instanceof NotEnrolledError) throw err;+ if (attempt) log(`push failed: ${err?.message || err}`);+ }+ await new Promise((r) => { setTimeout(r, 1500); });+ }+ return null;+ },+};++// ── browser ─────────────────────────────────────────────────────────────────+let browser = null;++async function ensureBrowser() {+ if (browser?.isConnected()) return browser;+ browser = await chromium.launch({+ headless: true,+ args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--mute-audio'],+ });+ browser.on('disconnected', () => {+ log('chromium disconnected; crawlers will restart on the next roster tick');+ for (const c of running.values()) c.stopped = true;+ running.clear();+ });+ log('chromium launched');+ return browser;+}++// ── orchestration ───────────────────────────────────────────────────────────+const running = new Map(); // agentId -> Crawler+let lastRosterError = null;+let lastRosterAt = 0;++function startCrawler(mission) {+ const crawler = new Crawler({ mission, browser, api, log, opts: OPTS });+ running.set(mission.agentId, crawler);+ log(`out reading: ${mission.name} on "${mission.topic}"`);+ crawler.start().finally(() => {+ if (running.get(mission.agentId) === crawler) running.delete(mission.agentId);+ });+}++async function tick() {+ let roster;+ try {+ roster = await api.roster(Math.max(CRAWLERS * 3, 24));+ lastRosterError = null;+ lastRosterAt = Date.now();+ } catch (err) {+ lastRosterError = err?.message || String(err);+ log(`roster unavailable: ${lastRosterError}`);+ return;+ }+ const enrolled = new Map(roster.map((m) => [m.agentId, m]));++ // Called home, or the topic changed.+ for (const [id, c] of running) {+ const m = enrolled.get(id);+ if (!m) {+ log(`called home: ${c.name}`);+ running.delete(id);+ c.stop();+ } else {+ c.updateMission(m);+ }+ }++ const waiting = roster.filter((m) => !running.has(m.agentId));+ // Shift change: if someone is waiting, the longest-running crawler past its+ // shift steps aside.+ if (waiting.length && running.size >= CRAWLERS) {+ const overdue = [...running.values()]+ .filter((c) => Date.now() - c.startedAt > SHIFT_MS)+ .sort((a, b) => a.startedAt - b.startedAt);+ for (const c of overdue.slice(0, waiting.length)) {+ log(`shift over: ${c.name}`);+ running.delete(c.agentId);+ c.stop();+ }+ }++ if (!waiting.length || running.size >= CRAWLERS) return;+ await ensureBrowser();+ for (const m of waiting) {+ if (running.size >= CRAWLERS) break;+ startCrawler(m);+ }+}++async function loop() {+ for (;;) {+ await tick().catch((err) => log(`tick failed: ${err?.stack || err}`));+ await new Promise((r) => { setTimeout(r, ROSTER_MS); });+ }+}++// ── liveness ────────────────────────────────────────────────────────────────+if (process.env.PORT) {+ http.createServer((req, res) => {+ res.writeHead(200, { 'content-type': 'application/json' });+ res.end(JSON.stringify({+ ok: true,+ crawlers: [...running.values()].map((c) => c.summary()),+ slots: CRAWLERS,+ browser: Boolean(browser?.isConnected()),+ lastRosterAt,+ lastRosterError,+ }));+ }).listen(Number(process.env.PORT), () => log(`liveness on :${process.env.PORT}`));+}++async function shutdown(signal) {+ log(`${signal}: calling every crawler home`);+ await Promise.allSettled([...running.values()].map((c) => c.stop()));+ await browser?.close().catch(() => {});+ process.exit(0);+}+process.on('SIGTERM', () => { shutdown('SIGTERM'); });+process.on('SIGINT', () => { shutdown('SIGINT'); });++log(`agent-crawler: ${CRAWLERS} slots against ${BASE_URL}`);+loop();diff --git a/workers/agent-crawler/net.js b/workers/agent-crawler/net.jsnew file mode 100644index 000000000..b1974e0d5--- /dev/null+++ b/workers/agent-crawler/net.js@@ -0,0 +1,116 @@+// Network fences for a browser that goes wherever links lead.+//+// guardHost(): every request the crawler's pages make is resolved first and+// refused when ANY address it resolves to is private, loopback, link-local or+// reserved. That is what keeps a hostile page (or a mission seed pointing at an+// internal name) from turning the crawler into a probe of the network it runs in.+//+// Robots: robots.txt is fetched once per origin and honored, including+// Crawl-delay, under the crawler's own product token.++import dns from 'node:dns/promises';+import net from 'node:net';+import robotsParser from 'robots-parser';++export const BOT_TOKEN = 'three.ws-crawler';+export const USER_AGENT =+ `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 ${BOT_TOKEN}/1.0 (+https://three.ws/crawl)`;++const blocked = new net.BlockList();+for (const [addr, prefix] of [+ ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8], ['169.254.0.0', 16], ['172.16.0.0', 12],+ ['192.0.0.0', 24], ['192.0.2.0', 24], ['192.168.0.0', 16], ['198.18.0.0', 15], ['198.51.100.0', 24],+ ['203.0.113.0', 24], ['224.0.0.0', 4], ['240.0.0.0', 4],+]) blocked.addSubnet(addr, prefix, 'ipv4');+for (const [addr, prefix] of [+ ['::', 128], ['::1', 128], ['::ffff:0:0', 96], ['64:ff9b::', 96], ['100::', 64], ['2001:db8::', 32],+ ['fc00::', 7], ['fe80::', 10], ['ff00::', 8],+]) blocked.addSubnet(addr, prefix, 'ipv6');++export function isBlockedAddress(ip) {+ const family = net.isIP(ip);+ if (!family) return true;+ return blocked.check(ip, family === 4 ? 'ipv4' : 'ipv6');+}++const INTERNAL_NAME = /(^|\.)(localhost|local|internal|intranet|lan|home|corp)$|^metadata(\.google\.internal)?$/i;+const hostCache = new Map(); // host -> { ok, at }+const HOST_TTL_MS = 10 * 60_000;++export async function guardHost(hostname) {+ const host = String(hostname || '').toLowerCase().replace(/^\[|\]$/g, '');+ if (!host || INTERNAL_NAME.test(host)) return false;+ if (net.isIP(host)) return !isBlockedAddress(host);+ const hit = hostCache.get(host);+ if (hit && Date.now() - hit.at < HOST_TTL_MS) return hit.ok;+ let ok = false;+ try {+ const addrs = await dns.lookup(host, { all: true, verbatim: true });+ ok = addrs.length > 0 && addrs.every((a) => !isBlockedAddress(a.address));+ } catch {+ ok = false;+ }+ hostCache.set(host, { ok, at: Date.now() });+ if (hostCache.size > 5000) hostCache.delete(hostCache.keys().next().value);+ return ok;+}++// fetch() for the crawler's own side requests (robots.txt, search reseeds),+// with the same host fence and a hard timeout.+export async function guardedFetch(url, { timeoutMs = 8000, headers = {} } = {}) {+ const u = new URL(url);+ if (!(await guardHost(u.hostname))) throw new Error(`refused private host ${u.hostname}`);+ return fetch(u, {+ headers: { 'user-agent': USER_AGENT, accept: '*/*', ...headers },+ redirect: 'follow',+ signal: AbortSignal.timeout(timeoutMs),+ });+}++const robotsCache = new Map(); // origin -> { robots, at }+const ROBOTS_TTL_MS = 60 * 60_000;++async function robotsFor(origin) {+ const hit = robotsCache.get(origin);+ if (hit && Date.now() - hit.at < ROBOTS_TTL_MS) return hit.robots;+ const robotsUrl = `${origin}/robots.txt`;+ let body = '';+ try {+ const res = await guardedFetch(robotsUrl, { timeoutMs: 6000, headers: { accept: 'text/plain' } });+ // RFC 9309: 4xx means no rules; 5xx or a network failure means "assume+ // disallowed for now" so a struggling server is not crawled harder.+ if (res.status >= 500) body = 'User-agent: *\nDisallow: /';+ else if (res.ok) body = (await res.text()).slice(0, 500_000);+ } catch {+ body = 'User-agent: *\nDisallow: /';+ }+ const robots = robotsParser(robotsUrl, body);+ robotsCache.set(origin, { robots, at: Date.now() });+ if (robotsCache.size > 3000) robotsCache.delete(robotsCache.keys().next().value);+ return robots;+}++// { allowed, delayMs } for one URL under the crawler's token.+export async function robotsCheck(url) {+ const u = new URL(url);+ const robots = await robotsFor(u.origin);+ const allowed = robots.isAllowed(u.href, BOT_TOKEN) !== false;+ const delay = Number(robots.getCrawlDelay(BOT_TOKEN));+ return { allowed, delayMs: Number.isFinite(delay) && delay > 0 ? Math.min(delay, 30) * 1000 : 0 };+}++// Per-domain politeness shared by every crawler in the process: at most one+// request to a domain every gapMs (or its robots Crawl-delay, if longer).+const domainNext = new Map();++export function domainReadyIn(host) {+ return Math.max(0, (domainNext.get(host) || 0) - Date.now());+}++export function claimDomain(host, gapMs) {+ domainNext.set(host, Date.now() + gapMs);+ if (domainNext.size > 10_000) {+ const now = Date.now();+ for (const [h, t] of domainNext) if (t < now) domainNext.delete(h);+ }+}diff --git a/workers/agent-crawler/package-lock.json b/workers/agent-crawler/package-lock.jsonnew file mode 100644index 000000000..3a056e42a--- /dev/null+++ b/workers/agent-crawler/package-lock.json@@ -0,0 +1,72 @@+{+ "name": "agent-crawler",+ "version": "0.1.0",+ "lockfileVersion": 3,+ "requires": true,+ "packages": {+ "": {+ "name": "agent-crawler",+ "version": "0.1.0",+ "dependencies": {+ "playwright": "1.61.1",+ "robots-parser": "^3.0.1"+ },+ "engines": {+ "node": ">=20"+ }+ },+ "node_modules/fsevents": {+ "version": "2.3.2",+ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",+ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",+ "hasInstallScript": true,+ "license": "MIT",+ "optional": true,+ "os": [+ "darwin"+ ],+ "engines": {+ "node": "^8.16.0 || ^10.6.0 || >=11.0.0"+ }+ },+ "node_modules/playwright": {+ "version": "1.61.1",+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz",+ "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==",+ "license": "Apache-2.0",+ "dependencies": {+ "playwright-core": "1.61.1"+ },+ "bin": {+ "playwright": "cli.js"+ },+ "engines": {+ "node": ">=18"+ },+ "optionalDependencies": {+ "fsevents": "2.3.2"+ }+ },+ "node_modules/playwright-core": {+ "version": "1.61.1",+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz",+ "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==",+ "license": "Apache-2.0",+ "bin": {+ "playwright-core": "cli.js"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "node_modules/robots-parser": {+ "version": "3.0.1",+ "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz",+ "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==",+ "license": "MIT",+ "engines": {+ "node": ">=10.0.0"+ }+ }+ }+}diff --git a/workers/agent-crawler/package.json b/workers/agent-crawler/package.jsonnew file mode 100644index 000000000..4cdd05560--- /dev/null+++ b/workers/agent-crawler/package.json@@ -0,0 +1,15 @@+{+ "name": "agent-crawler",+ "version": "0.1.0",+ "private": true,+ "type": "module",+ "main": "index.js",+ "scripts": {+ "start": "node index.js"+ },+ "engines": { "node": ">=20" },+ "dependencies": {+ "playwright": "1.61.1",+ "robots-parser": "^3.0.1"+ }+}diff --git a/workers/agent-crawler/seeds.js b/workers/agent-crawler/seeds.jsnew file mode 100644index 000000000..341a2f181--- /dev/null+++ b/workers/agent-crawler/seeds.js@@ -0,0 +1,69 @@+// Where an agent starts when its owner gave no start pages, or when it has+// walked itself into a dead end: real public search, tried in order until one+// answers. Each source is a free, keyless public endpoint.++import { guardedFetch } from './net.js';++// DuckDuckGo's HTML endpoint. Result anchors carry the target in the uddg+// parameter of a /l/ redirect. It sometimes answers bots with a 202 challenge+// page; that simply yields nothing and the next source takes over.+async function duckduckgo(topic) {+ const res = await guardedFetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(topic)}`, {+ headers: { accept: 'text/html' },+ });+ if (res.status !== 200) return [];+ const html = await res.text();+ const out = [];+ for (const m of html.matchAll(/class="result__a"[^>]*href="([^"]+)"/g)) {+ const href = m[1].replace(/&/g, '&');+ try {+ const u = new URL(href, 'https://duckduckgo.com');+ const target = u.searchParams.get('uddg') || (u.hostname.endsWith('duckduckgo.com') ? null : u.href);+ if (target && !/duckduckgo\.com\/y\.js/.test(target)) out.push(target);+ } catch { /* skip malformed result */ }+ }+ return out;+}++// Hacker News search (Algolia): stories carry the outside URL they link to,+// which is exactly the kind of page a reading agent should land on.+async function hackernews(topic) {+ const res = await guardedFetch(+ `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(topic)}&tags=story&hitsPerPage=20`,+ { headers: { accept: 'application/json' } },+ );+ if (!res.ok) return [];+ const data = await res.json();+ return (data.hits || []).map((h) => h.url).filter((u) => typeof u === 'string' && /^https?:\/\//.test(u));+}++// Wikipedia full-text search: always answers, always has something.+async function wikipedia(topic) {+ const res = await guardedFetch(+ `https://en.wikipedia.org/w/api.php?action=query&list=search&format=json&srlimit=8&srsearch=${encodeURIComponent(topic)}`,+ { headers: { accept: 'application/json' } },+ );+ if (!res.ok) return [];+ const data = await res.json();+ return (data.query?.search || []).map((s) => `https://en.wikipedia.org/wiki/${encodeURIComponent(s.title.replace(/ /g, '_'))}`);+}++export const SOURCES = [+ { name: 'duckduckgo', fn: duckduckgo },+ { name: 'hackernews', fn: hackernews },+ { name: 'wikipedia', fn: wikipedia },+];++// Gather start pages for a topic. Returns { urls, source } from the first source+// with results, so the crawler can say where its leads came from.+export async function searchSeeds(topic, log = () => {}) {+ for (const { name, fn } of SOURCES) {+ try {+ const urls = [...new Set(await fn(topic))].slice(0, 12);+ if (urls.length) return { urls, source: name };+ } catch (err) {+ log(`seed source ${name} failed: ${err?.message || err}`);+ }+ }+ return { urls: [], source: null };+}