Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// POST /api/avatar/video-generate
//
// Submits a talking-avatar video generation job to the LongCat-Video-Avatar-1.5
// Cloud Run worker. Returns immediately with a job_id for polling.
//
// Free-plan users get 1 lifetime generation. Paid users are unlimited.
//
// Request body (JSON):
// image_url string — publicly accessible reference image (PNG/JPG)
// audio_url string — publicly accessible audio file (WAV/MP3)
// prompt string? — optional text description
// avatar_id string? three.ws avatar id, resolved to a reference image
// when image_url is not supplied
//
// Response 202:
// { job_id, status: "queued" }
//
// GET returns an availability document instead of a job:
// { endpoint, available, reason, free_generations }
// so a client can tell a visitor the renderer is offline BEFORE it walks them
// through uploading audio. Without it the only signal was a 503 raised after
// the upload had already happened.
//
// Errors:
// 400 invalid_request - missing required fields
// 402 free_trial_used - free user has already used their 1 free generation
// 502 worker_error - Cloud Run worker returned an error
// 503 worker_unconfigured - LongCat worker is not configured on this deployment
import { cors, error, json, wrap, rateLimited } from '../_lib/http.js';
import { limits } from '../_lib/rate-limit.js';
import { sql } from '../_lib/db.js';
import { publicUrl, thumbnailUrl } from '../_lib/r2.js';
import { getSessionUser } from '../_lib/auth.js';
import { env } from '../_lib/env.js';
import { fetchUpstream } from '../_lib/upstream-fetch.js';
// A submit starts a GPU job, so it is never retried; the deadline only bounds
// the wait for the worker to accept it. Shared breaker with video-status.
const WORKER_BREAKER = 'longcat-worker';
const GENERATE_TIMEOUT_MS = 60_000;
import { resolveRenderParams, renderAvatarImage } from '../_lib/avatar-render.js';
// Rendering a fresh reference image runs headless chromium, so this handler can
// legitimately outrun the 10 s default on a first-time avatar.
export const maxDuration = 60;
// Mirror of trustedOrigin() in api/avatar/optimize.js: image_url/audio_url are
// forwarded to the LongCat worker, which fetches them server-side. Restrict them
// to https on a three.ws-controlled host (APP_ORIGIN / R2 public domain) so a
// caller can't drive the worker into an SSRF against internal/metadata hosts.
function trustedMediaUrl(url) {
let u;
try {
u = new URL(url);
} catch {
return false;
}
if (u.protocol !== 'https:') return false;
const allowed = new Set();
try {
if (env.APP_ORIGIN) allowed.add(new URL(env.APP_ORIGIN).host);
} catch {
/* ignore malformed env */
}
try {
if (env.S3_PUBLIC_DOMAIN) allowed.add(new URL(env.S3_PUBLIC_DOMAIN).host);
} catch {
/* ignore malformed env */
}
return allowed.has(u.host);
}
// Resolve the worker's address and credential together, returning null when
// either is unset. Both used to throw from inside the `fetch` try-block, so an
// unconfigured deployment answered `502 worker_unreachable` with the literal
// env-var name in the body: the wrong status (nothing was unreachable, the
// endpoint was never configured) and an operator detail the caller has no
// business seeing. Resolving up front turns that into the documented 503 and
// keeps the var name in the server log where it belongs.
function workerConfig() {
const url = process.env.LONGCAT_WORKER_URL;
const key = process.env.LONGCAT_WORKER_KEY;
if (!url || !key) return null;
return { url: url.replace(/\/$/, ''), key };
}
// The worker writes whatever this resolves to straight into `ref_image.png` and
// hands it to inference as `cond_image`, so it has to be an actual raster image.
// Returning the avatar's GLB (which is what this did) produced a job that always
// failed deep inside the worker, minutes after the caller was told "queued".
// Order: the stored thumbnail if one exists, otherwise a portrait render through
// the same cached pipeline /api/avatar/render uses, so the second video from an
// avatar costs a HEAD instead of a chromium launch.
const REFERENCE_RENDER = resolveRenderParams({
scene: 'portrait',
size: '768',
bg: '#101010',
format: 'png',
}).params;
async function resolveImageUrl(avatarId, requesterId) {
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(String(avatarId || ''))) {
throw Object.assign(new Error('avatar not found'), { code: 'avatar_not_found', status: 404 });
}
// Enforce ownership/visibility: the caller may only generate video from an
// avatar they own or one that is public/unlisted — never another user's
// private avatar. (Previously resolved by id alone — an IDOR.)
const rows = await sql`
select id, storage_key, thumbnail_key, updated_at from avatars
where id = ${avatarId} and deleted_at is null
and (owner_id = ${requesterId} or visibility in ('public', 'unlisted'))
limit 1
`;
if (!rows[0])
throw Object.assign(new Error('avatar not found'), {
code: 'avatar_not_found',
status: 404,
});
const stored = thumbnailUrl(rows[0].thumbnail_key);
if (stored) return stored;
try {
const out = await renderAvatarImage({
avatar: { id: rows[0].id, updated_at: rows[0].updated_at },
glbUrl: publicUrl(rows[0].storage_key),
params: REFERENCE_RENDER,
awaitUpload: true,
});
return out.imageUrl;
} catch (err) {
console.error('[video-generate] reference render failed:', err?.message || err);
throw Object.assign(
new Error('Could not render a reference image for this avatar. Give it a thumbnail and try again.'),
{ code: 'reference_image_failed', status: 502 },
);
}
}
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,POST,OPTIONS', credentials: true })) return;
if (req.method === 'GET') {
const configured = workerConfig() !== null;
return json(
res,
200,
{
endpoint: 'POST /api/avatar/video-generate',
description:
'Render a talking-head video from a three.ws avatar and a spoken audio clip.',
available: configured,
reason: configured ? 'ready' : 'worker_unconfigured',
free_generations: 1,
accepts: {
avatar_id: 'three.ws avatar UUID; resolved server-side to a reference image',
image_url: 'https URL on a three.ws host, used instead of avatar_id',
audio_url: 'https URL on a three.ws host (WAV/MP3/M4A)',
prompt: 'optional scene description',
},
},
{ 'cache-control': 'no-store' },
);
}
if (req.method !== 'POST')
return error(res, 405, 'method_not_allowed', `method ${req.method} not allowed`);
let session;
try {
session = await getSessionUser(req);
if (!session) throw new Error('no session');
} catch {
return error(res, 401, 'unauthorized', 'valid session required');
}
const userId = session.id ?? session.userId;
const [rl, rlg] = await Promise.all([
limits.videoGenerateUser(userId),
limits.videoGenerateGlobal(),
]);
if (!rl.success) return rateLimited(res, rl);
if (!rlg.success) return rateLimited(res, rlg);
// Check plan + usage. Free plan users get exactly 1 lifetime generation.
const [userRow] = await sql`
select plan from users where id = ${userId} and deleted_at is null limit 1
`;
const isFree = userRow?.plan === 'free';
// Fast-path rejection. The authoritative, race-safe gate is the reservation
// insert below — this early read just spares already-spent users the
// validation work.
if (isFree) {
const [usageRow] = await sql`
select count(*) as n from usage_events
where user_id = ${userId} and kind = 'video_generate'
`;
if (Number(usageRow?.n) >= 1) {
return error(
res,
402,
'free_trial_used',
'You have used your 1 free video. Upgrade to generate more.',
);
}
}
const body = req.body || {};
let { image_url: imageUrl, audio_url: audioUrl, avatar_id: avatarId, prompt } = body;
if (!audioUrl) return error(res, 400, 'invalid_request', 'audio_url is required');
if (!imageUrl && avatarId) {
try {
imageUrl = await resolveImageUrl(avatarId, userId);
} catch (err) {
return error(res, err.status || 400, err.code || 'invalid_request', err.message);
}
}
if (!imageUrl) return error(res, 400, 'invalid_request', 'image_url or avatar_id is required');
// The worker fetches these URLs server-side, so they must be https on a
// three.ws-controlled host. A server-resolved avatar render (publicUrl →
// R2 domain) already satisfies this; an arbitrary caller-supplied URL must
// pass the same allowlist or we'd hand the worker an SSRF primitive.
if (!trustedMediaUrl(imageUrl)) {
return error(res, 400, 'invalid_request', 'image_url must be an https three.ws-hosted URL');
}
if (!trustedMediaUrl(audioUrl)) {
return error(res, 400, 'invalid_request', 'audio_url must be an https three.ws-hosted URL');
}
// Validation first (a malformed request deserves an actionable 400, not a
// misleading "service unavailable"), then the worker-config gate, and only
// then the reservation. The free-trial reservation below is a real DB write,
// so a deployment that can never reach the worker must not insert and then
// delete a usage row on every attempt.
const worker = workerConfig();
if (!worker) {
console.error('[video-generate] LONGCAT_WORKER_URL / LONGCAT_WORKER_KEY not set on this deployment');
return error(
res,
503,
'worker_unconfigured',
'Talking-avatar video generation is not available on this deployment.',
);
}
// Atomically reserve the free-trial slot BEFORE submitting the worker job.
// The old check-then-insert straddled the worker call, so two concurrent
// requests could both pass the count check and both generate. Reserving
// first closes that window: usage_events.id is a bigserial, so after our
// reservation lands, any earlier (lower-id) video_generate row for this user
// means another request already holds or spent the single free slot — we
// release ours and reject. The row is deleted if worker submission fails,
// so a worker outage never burns the trial.
let reservationId = null;
if (isFree) {
const [reserved] = await sql`
insert into usage_events (user_id, kind, meta)
values (${userId}, 'video_generate', '{}'::jsonb)
returning id
`;
reservationId = reserved.id;
const [prior] = await sql`
select count(*) as n from usage_events
where user_id = ${userId} and kind = 'video_generate' and id < ${reservationId}
`;
if (Number(prior?.n) >= 1) {
await releaseReservation(reservationId);
return error(
res,
402,
'free_trial_used',
'You have used your 1 free video. Upgrade to generate more.',
);
}
}
let result;
try {
let workerRes;
try {
workerRes = await fetchUpstream(`${worker.url}/generate`, {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${worker.key}`,
},
body: JSON.stringify({
image_url: imageUrl,
audio_url: audioUrl,
prompt: prompt || 'A person talking naturally.',
}),
}, { name: WORKER_BREAKER, timeoutMs: GENERATE_TIMEOUT_MS, attempts: 1, okWhen: () => true });
} catch (err) {
await releaseReservation(reservationId);
return error(res, 502, 'worker_unreachable', err?.message || 'worker request failed');
}
if (!workerRes.ok) {
const text = await workerRes.text().catch(() => '');
await releaseReservation(reservationId);
return error(
res,
502,
'worker_error',
`worker returned ${workerRes.status}: ${text.slice(0, 200)}`,
);
}
result = await workerRes.json();
} catch (err) {
await releaseReservation(reservationId);
return error(
res,
502,
'worker_error',
err?.message || 'worker returned an unreadable response',
);
}
const jobId = result.job_id;
// Record usage so we can verify job ownership. The free-trial quota row
// already exists (the reservation) — attach the job id to it. Paid users
// get a best-effort audit row; a logging failure must not block them.
try {
if (reservationId != null) {
await sql`
update usage_events set meta = ${JSON.stringify({ job_id: jobId })}::jsonb
where id = ${reservationId}
`;
} else {
await sql`
insert into usage_events (user_id, kind, meta)
values (${userId}, 'video_generate', ${JSON.stringify({ job_id: jobId })}::jsonb)
`;
}
} catch (err) {
console.error('[video-generate] failed to record job id on usage event:', err);
}
return json(res, 202, { job_id: jobId, status: result.status });
});
// Best-effort delete of a free-trial reservation row after a failed worker
// submission. A delete failure leaves the slot consumed (fail-closed) rather
// than risking unlimited free generations.
async function releaseReservation(reservationId) {
if (reservationId == null) return;
try {
await sql`delete from usage_events where id = ${reservationId}`;
} catch (err) {
console.error('[video-generate] failed to release free-trial reservation:', err);
}
}