Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// The Crawl: three.ws agents reading the open web in real browsers, watched live.
// Doc: docs/crawl.md. Worker: workers/agent-crawler. Page: /crawl.
//
// Routes:
// GET /api/crawl/stats fleet + corpus totals, top domains (cached 15s)
// GET /api/crawl/crawlers enrolled crawlers with avatar + live state
// GET /api/crawl/live SSE: snapshot, then step/sleep/page events
// GET /api/crawl/frame?agent=<uuid> latest JPEG of that crawler's browser
// GET /api/crawl/pages?agent=&limit=&before=&format=json|jsonl
// the corpus, newest first (jsonl = dataset export)
// GET /api/crawl/mission?agent=<uuid> one agent's mission (public summary)
// PUT /api/crawl/mission owner sends an agent out { agentId, topic, seeds[], enabled }
// DELETE /api/crawl/mission?agent=<uuid> owner calls it home
// GET /api/crawl/mine the signed-in owner's agents + their missions
// GET /api/crawl/roster worker: who should be walking (bearer CRAWL_WORKER_SECRET)
// POST /api/crawl/push worker: one step (+ optional finished page) for one crawler
//
// Storage: Redis carries the live layer (crawl:live:<id> step JSON, crawl:frame:<id>
// base64 JPEG, crawl:active sorted set, crawl:feed recent reads); Postgres carries
// the roster (crawl_missions) and the corpus (crawl_pages); object storage holds
// each page's full cleaned text under crawl/text/.
import { timingSafeEqual } from 'node:crypto';
import { cors, error, json, method, rateLimited, readJson, wrap } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { getRedis } from '../_lib/redis.js';
import { sql } from '../_lib/db.js';
import { isUuid } from '../_lib/validate.js';
import { getSessionUser, extractBearer } from '../_lib/auth.js';
import { requireCsrf } from '../_lib/csrf.js';
import { objectStorageConfigured, putObject, publicUrlOrNull, thumbnailUrl } from '../_lib/r2.js';
import { resolveAvatarUrl } from '../_lib/avatars.js';
import {
CRAWL_LIVE_TTL, CRAWL_ACTIVE_WINDOW_MS, isFrameB64, pageMemory, sanitizePage, sanitizeStep, validateMission,
} from '../_lib/crawl.js';
export const maxDuration = 300;
const WORKER_SECRET = process.env.CRAWL_WORKER_SECRET || '';
const LIVE_KEY = (id) => `crawl:live:${id}`;
const FRAME_KEY = (id) => `crawl:frame:${id}`;
const ACTIVE_KEY = 'crawl:active';
const FEED_KEY = 'crawl:feed';
const STATS_KEY = 'crawl:stats:v1';
const FEED_CAP = 60;
const MEMORY_TTL_MS = 30 * 24 * 3600 * 1000;
const MEMORY_MIN_RELEVANCE = 0.12;
function isWorker(req) {
const bearer = extractBearer(req);
if (!bearer || WORKER_SECRET.length < 16) return false;
const a = Buffer.from(bearer);
const b = Buffer.from(WORKER_SECRET);
return a.length === b.length && timingSafeEqual(a, b);
}
// Upstash's REST client parses JSON values on read, so a record can come back as
// a string or an already-parsed object. Accept both.
function parseRecord(v) {
if (v && typeof v === 'object') return v;
if (typeof v !== 'string') return null;
try { return JSON.parse(v); } catch { return null; }
}
function avatarFields(row) {
const isPublic = row.avatar_visibility === 'public' || row.avatar_visibility === 'unlisted';
let thumb = null;
if (isPublic && row.thumbnail_key) {
try { thumb = thumbnailUrl(row.thumbnail_key); } catch { thumb = null; }
}
return { glbRow: isPublic && row.storage_key ? row : null, thumb };
}
async function resolveGlb(row) {
if (!row) return null;
try {
const { url } = await resolveAvatarUrl({
id: row.avatar_id,
storage_key: row.storage_key,
baked_storage_key: row.baked_storage_key,
appearance_hash: row.appearance_hash,
appearance: row.appearance,
visibility: row.avatar_visibility,
});
return url;
} catch {
return null;
}
}
async function awakeSet(r) {
if (!r) return new Map();
const now = Date.now();
try {
const raw = await r.zrange(ACTIVE_KEY, now - CRAWL_ACTIVE_WINDOW_MS, now, { byScore: true, withScores: true });
const out = new Map();
for (let i = 0; i + 1 < (raw || []).length; i += 2) out.set(String(raw[i]), Number(raw[i + 1]));
return out;
} catch {
return new Map();
}
}
// ── worker: roster ──────────────────────────────────────────────────────────
async function handleRoster(req, res) {
if (!isWorker(req)) return error(res, 401, 'unauthorized', 'crawl worker credential required');
const url = new URL(req.url, 'http://x');
const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 24));
const rows = await sql`
SELECT m.agent_id, m.topic, m.seeds, m.pages_read, i.name
FROM crawl_missions m
JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL
WHERE m.enabled = true
ORDER BY m.last_at ASC NULLS FIRST
LIMIT ${limit}
`;
return json(res, 200, {
crawlers: rows.map((r) => ({
agentId: r.agent_id, name: r.name, topic: r.topic, seeds: r.seeds || [], pagesRead: Number(r.pages_read) || 0,
})),
}, { 'cache-control': 'no-store' });
}
// ── worker: push ────────────────────────────────────────────────────────────
async function storeText(page) {
if (!page.text || !objectStorageConfigured()) return null;
const key = `crawl/text/${page.hash.slice(0, 2)}/${page.hash}.txt`;
try {
await putObject({ key, body: page.text, contentType: 'text/plain; charset=utf-8', metadata: { url: encodeURI(page.url).slice(0, 1024) } });
return key;
} catch (err) {
console.warn('[crawl] text store failed:', err?.message);
return null;
}
}
async function recordPage(agentId, mission, page) {
const [row] = await sql`
INSERT INTO crawl_pages (agent_id, url, url_hash, domain, title, gist, tokens, links_out, relevance, from_url)
VALUES (${agentId}, ${page.url}, ${page.hash}, ${page.domain}, ${page.title}, ${page.gist}, ${page.tokens},
${page.linksOut}, ${page.relevance}, ${page.fromUrl})
ON CONFLICT (agent_id, url_hash) DO NOTHING
RETURNING id
`;
if (!row) return { inserted: false };
const textKey = await storeText(page);
const [counts] = await sql`
UPDATE crawl_missions
SET pages_read = pages_read + 1, tokens_read = tokens_read + ${page.tokens},
last_url = ${page.url}, last_at = now(), updated_at = now()
WHERE agent_id = ${agentId}
RETURNING pages_read, tokens_read
`;
if (textKey) await sql`UPDATE crawl_pages SET text_key = ${textKey} WHERE id = ${row.id}`;
// The agent keeps what it read, but only when its own owner sent it out: an
// agent's memory is the owner's to shape.
const ownerSent = mission.set_by && mission.set_by === mission.user_id;
if (ownerSent && page.gist && (page.relevance == null || page.relevance >= MEMORY_MIN_RELEVANCE)) {
const salience = 0.25 + 0.4 * (page.relevance ?? 0.3);
const expires = new Date(Date.now() + MEMORY_TTL_MS).toISOString();
await sql`
INSERT INTO agent_memories (agent_id, type, content, tags, context, salience, tier, expires_at)
VALUES (${agentId}, 'reference', ${pageMemory(page, mission.topic)}, ${['crawl', page.domain]},
${JSON.stringify({ source: 'crawl', url: page.url, page_id: row.id })}::jsonb, ${salience}, 'archival', ${expires})
`.catch((err) => console.warn('[crawl] memory write failed:', err?.message));
}
return { inserted: true, id: row.id, pagesRead: Number(counts?.pages_read) || 0 };
}
async function handlePush(req, res) {
if (!isWorker(req)) return error(res, 401, 'unauthorized', 'crawl worker credential required');
let body;
try {
body = await readJson(req, 1_200_000);
} catch {
return error(res, 400, 'invalid_body', 'request body must be JSON under 1.2 MB');
}
const agentId = String(body?.agentId || '');
if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agentId must be a uuid');
const [mission] = await sql`
SELECT m.topic, m.set_by, m.enabled, m.pages_read, i.name, i.user_id
FROM crawl_missions m
JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL
WHERE m.agent_id = ${agentId}
LIMIT 1
`;
if (!mission || !mission.enabled) return error(res, 410, 'not_enrolled', 'this agent is not out crawling');
const step = sanitizeStep(body.step);
if (!step) return error(res, 400, 'invalid_step', 'step is required');
const page = body.page ? sanitizePage(body.page) : null;
const frame = isFrameB64(body.frame) ? body.frame : null;
let pagesRead = Number(mission.pages_read) || 0;
let recorded = null;
if (page) {
recorded = await recordPage(agentId, mission, page);
if (recorded.inserted) pagesRead = recorded.pagesRead;
}
const r = getRedis();
if (!r) return json(res, 200, { ok: true, live: false, pagesRead });
const now = Date.now();
const live = {
agentId, name: mission.name, topic: mission.topic, ts: now, pagesRead,
frameTs: frame ? now : undefined, ...step,
};
if (!frame) {
// A step without new pixels keeps pointing at the frame already stored.
const prev = parseRecord(await r.get(LIVE_KEY(agentId)).catch(() => null));
if (prev?.frameTs) live.frameTs = prev.frameTs;
}
const writes = [
r.set(LIVE_KEY(agentId), JSON.stringify(live), { ex: CRAWL_LIVE_TTL }),
r.zadd(ACTIVE_KEY, { score: now, member: agentId }),
r.zremrangebyscore(ACTIVE_KEY, 0, now - CRAWL_ACTIVE_WINDOW_MS * 2),
];
if (frame) writes.push(r.set(FRAME_KEY(agentId), frame, { ex: CRAWL_LIVE_TTL }));
// The live record keeps pointing at the last frame, so that frame must live
// exactly as long as the record does, or a crawler that rests on one page
// for longer than the TTL serves a frameTs whose image is gone.
else if (live.frameTs) writes.push(r.expire(FRAME_KEY(agentId), CRAWL_LIVE_TTL));
if (recorded?.inserted) {
const entry = { agentId, name: mission.name, url: page.url, title: page.title, domain: page.domain, tokens: page.tokens, gist: page.gist, ts: now };
writes.push(r.lpush(FEED_KEY, JSON.stringify(entry)).then(() => r.ltrim(FEED_KEY, 0, FEED_CAP - 1)));
}
await Promise.all(writes);
return json(res, 200, { ok: true, live: true, pagesRead, recorded: Boolean(recorded?.inserted) });
}
// ── public: frame ───────────────────────────────────────────────────────────
async function handleFrame(req, res) {
const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';
if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');
const r = getRedis();
if (!r) return error(res, 503, 'live_unavailable', 'the live layer is offline');
const b64 = await r.get(FRAME_KEY(agentId));
if (typeof b64 !== 'string' || !b64) {
return error(res, 404, 'asleep', 'this crawler has no live frame right now');
}
const buf = Buffer.from(b64, 'base64');
res.writeHead(200, {
'content-type': 'image/jpeg',
'content-length': buf.length,
// The page asks with ?t=<frameTs>, so each frame has its own URL.
'cache-control': 'public, max-age=30',
'x-content-type-options': 'nosniff',
});
res.end(buf);
}
// ── public: live SSE ────────────────────────────────────────────────────────
// One poller per server instance, fanned out to every connected viewer, so Redis
// load is flat in the number of viewers.
const hub = { subs: new Set(), timer: null, crawlers: new Map(), feedTs: 0, feed: [] };
const HUB_POLL_MS = 800;
async function readFleet(r) {
const awake = await awakeSet(r);
const ids = [...awake.keys()].slice(0, 120);
if (!ids.length) return [];
const raw = await r.mget(...ids.map(LIVE_KEY));
return (raw || []).map(parseRecord).filter(Boolean);
}
async function readFeed(r, n) {
const raw = await r.lrange(FEED_KEY, 0, n - 1).catch(() => []);
return (raw || []).map(parseRecord).filter(Boolean);
}
function broadcast(event, data) {
const line = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
for (const s of hub.subs) s(line);
}
async function hubTick() {
const r = getRedis();
if (!r || !hub.subs.size) return;
try {
const fleet = await readFleet(r);
const seen = new Set();
for (const c of fleet) {
seen.add(c.agentId);
const prev = hub.crawlers.get(c.agentId);
if (!prev || prev.ts !== c.ts) {
hub.crawlers.set(c.agentId, c);
broadcast('step', c);
}
}
for (const id of [...hub.crawlers.keys()]) {
if (!seen.has(id)) {
hub.crawlers.delete(id);
broadcast('sleep', { agentId: id });
}
}
const recent = await readFeed(r, 10);
const fresh = recent.filter((e) => e.ts > hub.feedTs).reverse();
for (const e of fresh) broadcast('page', e);
if (recent.length) hub.feedTs = Math.max(hub.feedTs, recent[0].ts);
} catch (err) {
console.warn('[crawl] hub tick failed:', err?.message);
}
}
function hubStart() {
if (hub.timer) return;
hub.timer = setInterval(hubTick, HUB_POLL_MS);
hub.timer.unref?.();
}
function hubStop() {
if (hub.subs.size || !hub.timer) return;
clearInterval(hub.timer);
hub.timer = null;
hub.crawlers.clear();
}
async function handleLive(req, res) {
const r = getRedis();
res.writeHead(200, {
'content-type': 'text/event-stream; charset=utf-8',
'cache-control': 'no-cache, no-transform',
connection: 'keep-alive',
'x-accel-buffering': 'no',
});
res.flushHeaders?.();
let open = true;
const write = (line) => {
if (!open || res.writableEnded) return;
try { res.write(line); } catch { open = false; }
};
if (!r) {
write(`event: snapshot\ndata: ${JSON.stringify({ crawlers: [], feed: [], live: false })}\n\n`);
} else {
const [fleet, feed] = await Promise.all([readFleet(r).catch(() => []), readFeed(r, 30)]);
for (const c of fleet) hub.crawlers.set(c.agentId, c);
if (feed.length) hub.feedTs = Math.max(hub.feedTs, feed[0].ts);
write(`event: snapshot\ndata: ${JSON.stringify({ crawlers: fleet, feed, live: true })}\n\n`);
}
hub.subs.add(write);
hubStart();
const ping = setInterval(() => write('event: ping\ndata: {}\n\n'), 15_000);
const end = setTimeout(() => { try { res.end(); } catch { /* closed */ } }, 280_000);
const close = () => {
open = false;
clearInterval(ping);
clearTimeout(end);
hub.subs.delete(write);
hubStop();
};
req.on('close', close);
res.on('close', close);
}
// ── public: stats ───────────────────────────────────────────────────────────
async function handleStats(req, res) {
const r = getRedis();
let cached = null;
if (r) cached = parseRecord(await r.get(STATS_KEY).catch(() => null));
if (!cached) {
const [[totals], [enrolled], top] = await Promise.all([
sql`
SELECT count(*)::bigint AS pages,
coalesce(sum(tokens), 0)::bigint AS tokens,
count(DISTINCT domain)::int AS domains,
count(DISTINCT url_hash)::bigint AS unique_pages,
count(*) FILTER (WHERE created_at > now() - interval '24 hours')::bigint AS pages_24h
FROM crawl_pages
`,
sql`SELECT count(*)::int AS n FROM crawl_missions WHERE enabled = true`,
sql`
SELECT domain, count(*)::int AS pages
FROM crawl_pages
WHERE created_at > now() - interval '7 days'
GROUP BY domain ORDER BY pages DESC LIMIT 12
`,
]);
cached = {
pagesRead: Number(totals.pages) || 0,
uniquePages: Number(totals.unique_pages) || 0,
tokens: Number(totals.tokens) || 0,
domains: totals.domains || 0,
pages24h: Number(totals.pages_24h) || 0,
crawlersEnrolled: enrolled.n || 0,
topDomains: top.map((t) => ({ domain: t.domain, pages: t.pages })),
};
if (r) await r.set(STATS_KEY, JSON.stringify(cached), { ex: 15 }).catch(() => {});
}
const awake = await awakeSet(r);
return json(res, 200, { ...cached, crawlersAwake: awake.size, ts: Date.now() }, { 'cache-control': 'public, max-age=10' });
}
// ── public: crawlers ────────────────────────────────────────────────────────
async function handleCrawlers(req, res) {
const r = getRedis();
const [rows, awake] = await Promise.all([
sql`
SELECT m.agent_id, m.topic, m.pages_read, m.tokens_read, m.last_url, m.last_at, m.enabled,
i.name, i.description,
a.id AS avatar_id, a.storage_key, a.baked_storage_key, a.appearance_hash, a.appearance,
a.thumbnail_key, a.visibility AS avatar_visibility
FROM crawl_missions m
JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL
LEFT JOIN avatars a ON a.id = i.avatar_id AND a.deleted_at IS NULL
WHERE m.enabled = true
ORDER BY m.last_at DESC NULLS LAST
LIMIT 120
`,
awakeSet(r),
]);
const crawlers = await Promise.all(rows.map(async (row) => {
const { glbRow, thumb } = avatarFields(row);
return {
agentId: row.agent_id,
name: row.name,
description: row.description ? String(row.description).slice(0, 200) : null,
topic: row.topic,
pagesRead: Number(row.pages_read) || 0,
tokensRead: Number(row.tokens_read) || 0,
lastUrl: row.last_url,
lastAt: row.last_at,
awake: awake.has(row.agent_id),
avatarUrl: await resolveGlb(glbRow),
thumbnail: thumb,
};
}));
crawlers.sort((a, b) => Number(b.awake) - Number(a.awake) || b.pagesRead - a.pagesRead);
return json(res, 200, { crawlers }, { 'cache-control': 'public, max-age=10' });
}
// ── public: pages (corpus) ──────────────────────────────────────────────────
async function handlePages(req, res) {
const p = new URL(req.url, 'http://x').searchParams;
const agent = p.get('agent');
if (agent && !isUuid(agent)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');
const limit = Math.min(500, Math.max(1, Number(p.get('limit')) || 40));
const before = Number(p.get('before'));
const beforeId = Number.isSafeInteger(before) && before > 0 ? before : null;
const rows = await sql`
SELECT p.id, p.agent_id, i.name AS agent_name, p.url, p.domain, p.title, p.gist, p.tokens,
p.links_out, p.relevance, p.from_url, p.text_key, p.created_at
FROM crawl_pages p
JOIN agent_identities i ON i.id = p.agent_id
WHERE (${agent}::uuid IS NULL OR p.agent_id = ${agent}::uuid)
AND (${beforeId}::bigint IS NULL OR p.id < ${beforeId}::bigint)
ORDER BY p.id DESC
LIMIT ${limit}
`;
const pages = rows.map((row) => ({
id: Number(row.id),
agentId: row.agent_id,
agentName: row.agent_name,
url: row.url,
domain: row.domain,
title: row.title,
gist: row.gist,
tokens: row.tokens,
linksOut: row.links_out,
relevance: row.relevance,
fromUrl: row.from_url,
textUrl: publicUrlOrNull(row.text_key),
readAt: row.created_at,
}));
if (p.get('format') === 'jsonl') {
res.writeHead(200, {
'content-type': 'application/x-ndjson; charset=utf-8',
'cache-control': 'public, max-age=30',
'content-disposition': 'inline; filename="three-ws-crawl.jsonl"',
});
res.end(pages.map((x) => JSON.stringify(x)).join('\n') + (pages.length ? '\n' : ''));
return;
}
const next = pages.length === limit ? pages[pages.length - 1].id : null;
return json(res, 200, { pages, next }, { 'cache-control': 'public, max-age=10' });
}
// ── mission (public read, owner write) ──────────────────────────────────────
async function ownedAgent(agentId, userId) {
const [row] = await sql`
SELECT id, name FROM agent_identities
WHERE id = ${agentId} AND user_id = ${userId} AND deleted_at IS NULL
LIMIT 1
`;
return row || null;
}
function missionView(row) {
return {
agentId: row.agent_id,
name: row.name,
topic: row.topic,
seeds: row.seeds || [],
enabled: row.enabled,
pagesRead: Number(row.pages_read) || 0,
tokensRead: Number(row.tokens_read) || 0,
lastUrl: row.last_url,
lastAt: row.last_at,
};
}
async function handleMission(req, res) {
if (req.method === 'GET') {
const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';
if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');
const [row] = await sql`
SELECT m.*, i.name FROM crawl_missions m
JOIN agent_identities i ON i.id = m.agent_id AND i.deleted_at IS NULL
WHERE m.agent_id = ${agentId} LIMIT 1
`;
if (!row) return json(res, 200, { mission: null }, { 'cache-control': 'no-store' });
return json(res, 200, { mission: missionView(row) }, { 'cache-control': 'no-store' });
}
const auth = await getSessionUser(req, res);
const userId = auth?.id;
if (!userId) return error(res, 401, 'unauthorized', 'sign in to send your agent out');
if (!(await requireCsrf(req, res, userId))) return;
const rl = await limits.crawlMissionIp(clientIp(req));
if (!rl.success) return rateLimited(res, rl);
if (req.method === 'DELETE') {
const agentId = new URL(req.url, 'http://x').searchParams.get('agent') || '';
if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agent must be a uuid');
if (!(await ownedAgent(agentId, userId))) return error(res, 403, 'forbidden', 'you do not own this agent');
await sql`UPDATE crawl_missions SET enabled = false, updated_at = now() WHERE agent_id = ${agentId}`;
const r = getRedis();
if (r) await Promise.all([r.del(LIVE_KEY(agentId)), r.zrem(ACTIVE_KEY, agentId)]).catch(() => {});
return json(res, 200, { ok: true, enabled: false });
}
let body;
try { body = await readJson(req, 16_000); } catch { return error(res, 400, 'invalid_body', 'body must be JSON'); }
const agentId = String(body?.agentId || '');
if (!isUuid(agentId)) return error(res, 400, 'invalid_agent_id', 'agentId must be a uuid');
const agent = await ownedAgent(agentId, userId);
if (!agent) return error(res, 403, 'forbidden', 'you do not own this agent');
const m = validateMission(body);
if (m.error) return error(res, 400, 'invalid_mission', m.error);
const [row] = await sql`
INSERT INTO crawl_missions (agent_id, topic, seeds, enabled, set_by)
VALUES (${agentId}, ${m.topic}, ${m.seeds}, ${m.enabled}, ${userId})
ON CONFLICT (agent_id) DO UPDATE
SET topic = excluded.topic, seeds = excluded.seeds, enabled = excluded.enabled,
set_by = excluded.set_by, updated_at = now()
RETURNING *
`;
return json(res, 200, { mission: missionView({ ...row, name: agent.name }) });
}
async function handleMine(req, res) {
const auth = await getSessionUser(req, res);
const userId = auth?.id;
if (!userId) return error(res, 401, 'unauthorized', 'sign in to see your agents');
const rows = await sql`
SELECT i.id, i.name, a.thumbnail_key, a.visibility AS avatar_visibility,
m.topic, m.seeds, m.enabled, m.pages_read, m.tokens_read, m.last_url, m.last_at
FROM agent_identities i
LEFT JOIN avatars a ON a.id = i.avatar_id AND a.deleted_at IS NULL
LEFT JOIN crawl_missions m ON m.agent_id = i.id
WHERE i.user_id = ${userId} AND i.deleted_at IS NULL
ORDER BY m.enabled DESC NULLS LAST, i.created_at DESC
LIMIT 60
`;
return json(res, 200, {
agents: rows.map((row) => ({
agentId: row.id,
name: row.name,
thumbnail: avatarFields(row).thumb,
mission: row.topic ? missionView({ ...row, agent_id: row.id }) : null,
})),
}, { 'cache-control': 'no-store' });
}
const ROUTES = {
roster: { methods: ['GET'], fn: handleRoster },
push: { methods: ['POST'], fn: handlePush },
frame: { methods: ['GET'], fn: handleFrame },
live: { methods: ['GET'], fn: handleLive },
stats: { methods: ['GET'], fn: handleStats },
crawlers: { methods: ['GET'], fn: handleCrawlers },
pages: { methods: ['GET'], fn: handlePages },
mission: { methods: ['GET', 'PUT', 'DELETE'], fn: handleMission },
mine: { methods: ['GET'], fn: handleMine },
};
export default wrap(async function handleCrawl(req, res) {
const action = new URL(req.url, 'http://x').searchParams.get('action') || '';
const route = ROUTES[action];
if (cors(req, res, { methods: `${route ? route.methods.join(',') : 'GET'},OPTIONS` })) return;
if (!route) return error(res, 404, 'not_found', 'unknown crawl action');
if (!method(req, res, route.methods)) return;
// Worker traffic is authenticated by secret and paced by the worker itself.
// Frames get a wider bucket (a wall of tiles fetches one per step each).
if (action === 'frame') {
const rl = await limits.crawlFrameIp(clientIp(req));
if (!rl.success) return rateLimited(res, rl);
} else if (action !== 'push' && action !== 'roster') {
const rl = await limits.crawlReadIp(clientIp(req));
if (!rl.success) return rateLimited(res, rl);
}
return route.fn(req, res);
});