Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

10.1 KB
Raw Blame
// POST /api/agent-screen-push — agent pushes a screen frame to the live feed.
//
// Agents (snipers, browser workers, autonomous scripts) POST here to publish
// what they're currently doing. Viewers subscribe via /api/agent-screen-stream.
//
// Body (JSON):
//   agentId   string   — UUID of the agent
//   frame     object   — { data?, activity, type }
//     data       string?  — base64 data URL (data:image/png;base64,…) or omit for text-only
//     activity   string   — plain-language description of what the agent is doing
//     type       string   — "screenshot" | "activity" | "trade" | "analysis"
//
// Auth: bearer JWT (same as other agent-authenticated endpoints). The agentId
// in the body must match the token's sub or an agent the user owns. Workers
// running as the agent itself use the agent's own JWT.
//
// Storage: Upstash Redis, key agent:screen:{agentId}:frame (TTL 90s — if the
// agent stops pushing, the stream goes dark automatically). A secondary list
// agent:screen:{agentId}:log holds the last 50 activity entries (no image data)
// for the activity log panel.

import { timingSafeEqual } from 'node:crypto';
import { cors, error, json, method, rateLimited, readJson, wrap } from './_lib/http.js';
import { getSessionUser, authenticateBearer, extractBearer } from './_lib/auth.js';
import { limits, clientIp } from './_lib/rate-limit.js';
import { getRedis } from './_lib/redis.js';
import { sql } from './_lib/db.js';
import { isUuid } from './_lib/validate.js';
import { sanitizeFrameMeta } from '../src/shared/forge-frames.js';

// First-party on-demand caster pool (workers/agent-screen-pool) authenticates
// with a single shared secret and may push frames for ANY agent — it casts
// whichever agents viewers are currently watching, not agents it "owns". The
// secret is never optional-by-default: an unset/blank env disables the path.
const WORKER_SECRET = process.env.SCREEN_WORKER_SECRET || '';
function isPoolWorker(bearer) {
	if (!bearer || !WORKER_SECRET || WORKER_SECRET.length < 16) return false;
	const a = Buffer.from(bearer);
	const b = Buffer.from(WORKER_SECRET);
	return a.length === b.length && timingSafeEqual(a, b);
}

const FRAME_TTL = 90; // seconds — stream goes dark if agent stops pushing
const LOG_CAP = 50; // activity log entries kept per agent
const DATA_MAX = 800_000; // ~600 KB base64 PNG max (approx 450 KB PNG)
const ACTIVITY_MAX = 320; // chars

// Accept only raster image data URLs. SVG is rejected on purpose: it can carry
// active content (scripts, external fetches) that would execute the moment a
// viewer ever inline-rendered a frame instead of using <img src>. Anything that
// isn't a base64 raster image is dropped to text-only.
const RASTER_DATA_URL = /^data:image\/(png|jpeg|jpg|webp|gif);base64,[A-Za-z0-9+/=]+$/;
export function isRasterDataUrl(s) {
	return typeof s === 'string' && RASTER_DATA_URL.test(s);
}

// Whitelist + coerce a trade PnL ride-along payload. Keeps only known fields,
// drops non-finite numbers, and bounds the strings so an agent can't smuggle
// arbitrary data through the frame's `pnl` slot. Returns null when unusable.
const PNL_PHASES = ['scored', 'buy', 'hold', 'exit'];
export function sanitizePnl(pnl) {
	if (!pnl || typeof pnl !== 'object' || !PNL_PHASES.includes(pnl.phase)) return null;
	const out = { phase: pnl.phase };
	if (typeof pnl.mint === 'string') out.mint = pnl.mint.slice(0, 64);
	if (typeof pnl.symbol === 'string') out.symbol = pnl.symbol.slice(0, 32);
	for (const k of ['solDelta', 'pct', 'realizedUsd', 'unrealizedUsd']) {
		const n = Number(pnl[k]);
		if (Number.isFinite(n)) out[k] = n;
	}
	return out;
}

// Whitelist + coerce a market-maker `mm` ride-along (Floor Defense). Same shape
// the agent-mm worker writes directly to Redis, accepted here so any first-party
// pusher (or a JWT-authed owner) can drive the arena floor line through the HTTP
// transport too. Bounds the strings + drops non-finite numbers so a malformed
// push can't poison the stream. Returns null when unusable.
const MM_TYPES = ['mm_seed', 'mm_defend', 'mm_recycle', 'mm_rebalance', 'mm_graduate', 'mm_quote'];
export function sanitizeMm(mm) {
	if (!mm || typeof mm !== 'object' || !MM_TYPES.includes(mm.type)) return null;
	const out = { type: mm.type };
	for (const k of ['floorSol', 'priceSol', 'sizeSol']) {
		const n = Number(mm[k]);
		out[k] = Number.isFinite(n) && n >= 0 ? n : 0;
	}
	out.sideBuy = mm.sideBuy === true ? true : mm.sideBuy === false ? false : null;
	out.simulate = !!mm.simulate;
	if (typeof mm.mint === 'string') out.mint = mm.mint.slice(0, 64);
	if (typeof mm.signature === 'string') out.signature = mm.signature.slice(0, 96);
	return out;
}

// A pixels-only push (a periodic screenshot with no activity text) belongs in
// the frame slot, never in the 50-entry activity log: services/agent-screen-caster
// pushes one every FRAME_INTERVAL_MS, so logging them evicts the agent's real
// narration within seconds and leaves watchers reading blank lines. Log an entry
// only when it carries something to read: activity text, or a structured
// ride-along (PnL / forge meta / market-maker) the viewer renders on its own.
export function shouldLogEntry({ activity, pnl, meta, mm }) {
	return Boolean(activity || pnl || meta || mm);
}

export default wrap(async function handleAgentScreenPush(req, res) {
	if (cors(req, res, { methods: 'POST,OPTIONS' })) return;
	if (!method(req, res, ['POST'])) return;

	// Auth: first-party pool worker (shared secret, casts any agent) OR the
	// agent owner (bearer JWT / session, casts only their own agents).
	let userId = null;
	let isWorker = false;
	const bearer = extractBearer(req);
	if (isPoolWorker(bearer)) {
		isWorker = true;
	} else {
		if (bearer) {
			const auth = await authenticateBearer(bearer).catch(() => null);
			if (auth?.userId) userId = auth.userId;
		}
		if (!userId) {
			const auth = await getSessionUser(req, res);
			if (auth?.id) userId = auth.id;
		}
		if (!userId) return error(res, 401, 'unauthorized', 'authentication required');
	}

	// Rate limit: 6 frames/second per IP (generous for screenshot streams)
	const rl = await limits.apiIp(clientIp(req), { limit: 360, window: '60s' });
	if (!rl.success) return rateLimited(res, rl, 'frame rate limit exceeded');

	// Cap the body so a client can't force us to buffer tens of MB before we slice
	// the frame down to DATA_MAX. 1 MB comfortably holds a ~600 KB base64 frame.
	let body;
	try {
		body = await readJson(req, 1_100_000);
	} catch {
		return error(res, 400, 'invalid_body', 'request body must be valid JSON under 1 MB');
	}

	const { agentId, frame } = body || {};
	if (!agentId || typeof agentId !== 'string') {
		return error(res, 400, 'missing_agent_id', 'agentId is required');
	}
	// agent_identities.id is a uuid column, so a non-uuid throws 22P02 on the
	// ownership lookup below and a caster typo becomes a 500 instead of the 400 it
	// is. Reject the shape before it reaches Postgres.
	if (!isUuid(agentId)) {
		return error(res, 400, 'invalid_agent_id', 'agentId must be a uuid');
	}
	if (!frame || typeof frame !== 'object') {
		return error(res, 400, 'missing_frame', 'frame object is required');
	}

	const activity = String(frame.activity || '').slice(0, ACTIVITY_MAX);
	const type = ['screenshot', 'activity', 'trade', 'analysis'].includes(frame.type)
		? frame.type : 'activity';
	const sliced = typeof frame.data === 'string' ? frame.data.slice(0, DATA_MAX) : null;
	const data = sliced && isRasterDataUrl(sliced) ? sliced : null;
	// Optional structured PnL ride-along on 'trade' frames (Live Trading Desk).
	// Drives the viewer's PnL ticker + avatar emote; sanitized so a malformed
	// push can't poison the stream. Persisted on the frame AND the log entry so
	// the ticker survives a reconnect's backfill.
	const pnl = sanitizePnl(frame.pnl);
	// Optional forge sidecar on the final 'analysis' frame of a Live Avatar Forge
	// run: carries the durable GLB url + viewer link so every connected viewer can
	// load and animate the freshly-forged avatar. Sanitized + http(s)-gated so a
	// push can't smuggle a junk or javascript: url to the viewer's loader.
	const meta = sanitizeFrameMeta(frame.meta);
	// Optional market-maker ride-along (Floor Defense): the structured floor/price
	// + fill context the arena reads to draw the floor line and route emotes.
	const mm = sanitizeMm(frame.mm);

	// Ownership: the pool worker may cast any existing agent; owners are limited
	// to their own agents.
	const [agentRow] = isWorker
		? await sql`SELECT id FROM agent_identities WHERE id = ${agentId} AND deleted_at IS NULL LIMIT 1`
		: await sql`SELECT id FROM agent_identities WHERE id = ${agentId} AND user_id = ${userId} LIMIT 1`;
	if (!agentRow) return error(res, isWorker ? 404 : 403, isWorker ? 'not_found' : 'forbidden', isWorker ? 'agent not found' : 'agent not owned by this user');

	const r = getRedis();
	if (!r) return error(res, 503, 'redis_unavailable', 'frame store offline');

	const now = Date.now();
	const frameKey = `agent:screen:${agentId}:frame`;
	const logKey = `agent:screen:${agentId}:log`;

	// Full frame record (includes image data when present)
	const frameRecord = JSON.stringify({ ts: now, data, activity, type, agentId, ...(pnl ? { pnl } : {}), ...(meta ? { meta } : {}), ...(mm ? { mm } : {}) });

	// Log entry (no image — the activity log only needs text + metadata)
	const logEntry = JSON.stringify({ ts: now, activity, type, ...(pnl ? { pnl } : {}), ...(meta ? { meta } : {}), ...(mm ? { mm } : {}) });

	const logs = shouldLogEntry({ activity, pnl, meta, mm });

	await Promise.all([
		r.set(frameKey, frameRecord, { ex: FRAME_TTL }),
		...(logs ? [
			r.lpush(logKey, logEntry).then(() => r.ltrim(logKey, 0, LOG_CAP - 1)),
			r.expire(logKey, FRAME_TTL * 5),
		] : []),
		// Track active agents in a sorted set (score = timestamp) so the walk
		// scene can discover which agents have live streams to show desks for.
		r.zadd('agent:screen:active', { score: now, member: agentId }),
		// Evict agents that haven't pushed in 120s so the set can't grow unbounded
		// under continuous traffic (the read side filters by score, but never removes).
		r.zremrangebyscore('agent:screen:active', 0, now - 120_000),
		r.expire('agent:screen:active', FRAME_TTL * 3),
	]);

	return json(res, 200, { ok: true, ts: now });
});