Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
"""
LongCat Video Avatar service: audio-driven talking avatar video generation.
POST /generate { image_url, audio_url, prompt?, job_id? }
-> 202 { job_id, status: "queued" }
GET /jobs/:id -> { job_id, status, progress?, segments?, audio_seconds?,
video_url?, error?, updated_at }
GET /health -> { ok, model_loaded, missing_weights, resolution, ... }
Environment variables (all required unless noted):
API_KEY shared bearer secret (set via GCP Secret Manager)
GCS_BUCKET Cloud Storage bucket for output MP4s
FIRESTORE_PROJECT GCP project hosting Firestore
WEIGHTS_DIR root holding both LongCat repos (default: /weights/longcat,
see model_weights.py for the required layout)
LONGCAT_REPO_DIR path to cloned LongCat repo (default: /longcat)
MAX_CONCURRENT parallel inference jobs (default: 1; GPU-bound)
MAX_SEGMENTS cap on generated segments (default: 8, about 26 s of video)
RESOLUTION 480p or 720p (default: 720p)
"""
from __future__ import annotations
import asyncio
import base64
import json
import logging
import os
import shutil
import subprocess
import tempfile
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from pathlib import Path
import httpx
from fastapi import BackgroundTasks, FastAPI, HTTPException, Header
from google.cloud import firestore, storage
from pydantic import BaseModel
import model_weights
from inference_plan import (
ProgressTracker,
segment_span_seconds,
segments_for_audio,
select_output_video,
)
from worker_security import (
UnsafeUrlError,
fetch_remote_bytes_async,
require_api_key,
safe_error,
)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
log = logging.getLogger("main")
# ── config ─────────────────────────────────────────────────────────────────────
API_KEY = os.environ["API_KEY"]
GCS_BUCKET = os.environ["GCS_BUCKET"]
FIRESTORE_PROJECT = os.environ["FIRESTORE_PROJECT"]
WEIGHTS_DIR = Path(os.environ.get("WEIGHTS_DIR", "/weights/longcat"))
LONGCAT_REPO_DIR = Path(os.environ.get("LONGCAT_REPO_DIR", "/longcat"))
MAX_CONCURRENT = max(1, int(os.environ.get("MAX_CONCURRENT", "1")))
MAX_SEGMENTS = max(1, int(os.environ.get("MAX_SEGMENTS", "8")))
RESOLUTION = os.environ.get("RESOLUTION", "720p")
# Upstream's argparse restricts --resolution to these two. Catching a typo here
# turns a 40-minute-late CUDA crash into a boot failure the deploy can see.
SUPPORTED_RESOLUTIONS = ("480p", "720p")
if RESOLUTION not in SUPPORTED_RESOLUTIONS:
raise RuntimeError(
f"RESOLUTION={RESOLUTION!r} is not supported; expected one of {SUPPORTED_RESOLUTIONS}"
)
CHECKPOINT_DIR = model_weights.avatar_dir(WEIGHTS_DIR)
BASE_MODEL_DIR = model_weights.base_dir(WEIGHTS_DIR)
INFERENCE_SCRIPT = "run_demo_avatar_single_audio_to_video.py"
# ── global state ───────────────────────────────────────────────────────────────
_db: firestore.Client | None = None
_bucket: storage.Bucket | None = None
_job_sem: asyncio.Semaphore | None = None
_http: httpx.AsyncClient | None = None
# ── lifespan ───────────────────────────────────────────────────────────────────
@asynccontextmanager
async def lifespan(app: FastAPI):
global _db, _bucket, _job_sem, _http
_job_sem = asyncio.Semaphore(MAX_CONCURRENT)
# follow_redirects MUST stay False: redirect hops are re-validated against
# the SSRF allow-rules by fetch_remote_bytes_async, not by httpx.
_http = httpx.AsyncClient(follow_redirects=False, timeout=300)
_db = firestore.Client(project=FIRESTORE_PROJECT)
_bucket = storage.Client().bucket(GCS_BUCKET)
missing = model_weights.missing_paths(WEIGHTS_DIR)
if missing:
# Deliberately not downloading here. The required set is ~45 GB across
# two HuggingFace repos; pulling it inside the startup hook guarantees a
# startup-probe timeout and re-pulls on every cold start. Weights are
# staged into the GCS weights bucket once (stage-weights.sh) and mounted.
log.error(
"Model weights incomplete under %s: /generate will answer 503. Missing: %s. "
"Stage them with workers/longcat/stage-weights.sh.",
WEIGHTS_DIR,
model_weights.describe_missing(missing),
)
else:
log.info("Model weights complete under %s", WEIGHTS_DIR)
log.info(
"Server ready: resolution=%s, max_concurrent=%d, max_segments=%d, weights=%s",
RESOLUTION, MAX_CONCURRENT, MAX_SEGMENTS, WEIGHTS_DIR,
)
yield
await _http.aclose()
app = FastAPI(title="longcat-video-avatar", lifespan=lifespan)
# ── auth ───────────────────────────────────────────────────────────────────────
def _require_api_key(authorization: str | None) -> None:
try:
require_api_key(authorization, API_KEY)
except PermissionError as exc:
raise HTTPException(status_code=401, detail=str(exc)) from exc
# ── Firestore helpers ──────────────────────────────────────────────────────────
def _set_job(job_id: str, data: dict) -> None:
_db.collection("longcat_video_jobs").document(job_id).set(
{**data, "updated_at": datetime.now(timezone.utc)},
merge=True,
)
def _get_job(job_id: str) -> dict | None:
doc = _db.collection("longcat_video_jobs").document(job_id).get()
return doc.to_dict() if doc.exists else None
# ── inference worker ───────────────────────────────────────────────────────────
def _probe_audio_seconds(path: Path) -> float | None:
"""Duration of ``path`` per ffprobe, or None when it cannot be determined."""
try:
completed = subprocess.run(
[
"ffprobe", "-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
str(path),
],
capture_output=True,
text=True,
timeout=60,
check=True,
)
return float(completed.stdout.strip())
except (subprocess.SubprocessError, OSError, ValueError) as exc:
# OSError covers a missing ffprobe binary. Sizing falls back to a single
# segment, which is upstream's default: a shorter video beats a dead job.
log.warning("ffprobe could not read a duration from %s: %s", path, exc)
return None
async def _process_job(
job_id: str,
image_url: str,
audio_url: str,
prompt: str,
) -> None:
async with _job_sem:
_set_job(job_id, {"status": "running", "progress": 0.0})
workdir = Path(tempfile.mkdtemp(prefix=f"longcat_{job_id}_"))
try:
log.info("[%s] downloading inputs", job_id)
image_path = workdir / "ref_image.png"
audio_path = workdir / "audio.wav"
output_dir = workdir / "output"
output_dir.mkdir()
await _download_file(image_url, image_path)
await _download_file(audio_url, audio_path)
# One segment covers 3.72 s. Sizing from the real audio length is
# what stops a 20-second voice clip rendering as a 4-second video.
audio_seconds = await asyncio.get_event_loop().run_in_executor(
None, lambda: _probe_audio_seconds(audio_path)
)
segments = segments_for_audio(audio_seconds, MAX_SEGMENTS)
log.info(
"[%s] audio=%.2fs → %d segment(s) covering %.2fs",
job_id,
audio_seconds if audio_seconds is not None else -1.0,
segments,
segment_span_seconds(segments),
)
_set_job(job_id, {"audio_seconds": audio_seconds, "segments": segments})
config = {
"prompt": prompt,
"cond_image": str(image_path),
"cond_audio": {"person1": str(audio_path)},
}
config_path = workdir / "config.json"
config_path.write_text(json.dumps(config))
log.info("[%s] starting inference", job_id)
loop = asyncio.get_event_loop()
await loop.run_in_executor(
None,
lambda: _run_inference(job_id, config_path, output_dir, segments),
)
names = sorted(
p.relative_to(output_dir).as_posix() for p in output_dir.rglob("*.mp4")
)
chosen = select_output_video(names)
if chosen is None:
raise RuntimeError(
f"inference produced no final MP4 (saw {len(names)} candidate file(s))"
)
mp4_path = output_dir / chosen
log.info(
"[%s] uploading %s (%d bytes) to GCS",
job_id, chosen, mp4_path.stat().st_size,
)
blob_name = f"avatar-videos/{job_id}.mp4"
blob = _bucket.blob(blob_name)
blob.content_type = "video/mp4"
await loop.run_in_executor(
None,
lambda: blob.upload_from_filename(str(mp4_path)),
)
video_url = f"https://storage.googleapis.com/{GCS_BUCKET}/{blob_name}"
_set_job(job_id, {"status": "done", "progress": 1.0, "video_url": video_url})
log.info("[%s] done → %s", job_id, video_url)
except Exception as exc:
_set_job(job_id, {
"status": "failed",
"error": safe_error(exc, context=f"[{job_id}] pipeline"),
})
finally:
shutil.rmtree(workdir, ignore_errors=True)
def _run_inference(
job_id: str,
config_path: Path,
output_dir: Path,
segments: int,
) -> None:
cmd = [
"torchrun",
"--nproc_per_node=1",
INFERENCE_SCRIPT,
"--input_json", str(config_path),
"--output_dir", str(output_dir),
"--checkpoint_dir", str(CHECKPOINT_DIR),
"--resolution", RESOLUTION,
"--model_type", "avatar-v1.5",
# ai2v is audio-image-to-video: it consumes config.cond_image. at2v
# ignores the reference image entirely, which is never what this service
# wants. Upstream's default happens to be ai2v; pin it so an upstream
# default change cannot silently drop the caller's avatar.
"--stage_1", "ai2v",
"--num_segments", str(segments),
"--use_distill",
"--use_int8",
]
log.info("[%s] cmd: %s", job_id, " ".join(cmd))
# Stream stdout+stderr so we can parse progress in real time.
process = subprocess.Popen(
cmd,
cwd=str(LONGCAT_REPO_DIR),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
bufsize=1,
)
stdout_lines: list[str] = []
tracker = ProgressTracker(expected_segments=segments)
for raw_line in process.stdout:
line = raw_line.rstrip()
stdout_lines.append(line)
# Keep only enough tail for a failure report; a multi-segment 720p run
# emits tens of thousands of tqdm lines.
if len(stdout_lines) > 200:
del stdout_lines[:100]
log.info("[%s] %s", job_id, line)
progress = tracker.update(line)
if progress is not None:
_set_job(job_id, {"progress": progress})
process.wait()
if process.returncode != 0:
tail = "\n".join(stdout_lines[-100:])
raise RuntimeError(
f"torchrun failed (exit {process.returncode}):\n{tail}"
)
async def _download_file(url: str, dest: Path) -> None:
if url.startswith("data:"):
# Inline data URI: decode base64 directly, no HTTP request needed.
try:
_, encoded = url.split(",", 1)
dest.write_bytes(base64.b64decode(encoded))
except Exception as exc:
raise RuntimeError(f"failed to decode data URI: {exc}") from exc
return
# SSRF-hardened: https-only, private/loopback/link-local/metadata IPs are
# rejected after DNS resolution, redirects are re-validated per hop, and the
# response is size-bounded.
try:
data = await fetch_remote_bytes_async(_http, url)
except UnsafeUrlError as exc:
raise RuntimeError(f"refused to fetch url: {exc}") from exc
dest.write_bytes(data)
# ── routes ─────────────────────────────────────────────────────────────────────
class GenerateRequest(BaseModel):
image_url: str
audio_url: str
prompt: str = "A person talking naturally."
job_id: str | None = None
@app.post("/generate", status_code=202)
async def generate(
body: GenerateRequest,
background_tasks: BackgroundTasks,
authorization: str | None = Header(default=None),
) -> dict:
_require_api_key(authorization)
# Fail fast and loudly instead of queueing a job that will spend two minutes
# loading models and then die on a missing checkpoint.
missing = model_weights.missing_paths(WEIGHTS_DIR)
if missing:
log.error(
"rejecting /generate: weights incomplete under %s (%s)",
WEIGHTS_DIR, model_weights.describe_missing(missing),
)
raise HTTPException(
status_code=503,
detail="model weights are not staged on this instance",
)
job_id = body.job_id or str(uuid.uuid4())
_set_job(job_id, {
"job_id": job_id,
"status": "queued",
"progress": None,
"image_url": body.image_url,
"audio_url": body.audio_url,
"prompt": body.prompt,
"created_at": datetime.now(timezone.utc),
})
background_tasks.add_task(
_process_job, job_id, body.image_url, body.audio_url, body.prompt,
)
return {"job_id": job_id, "status": "queued"}
@app.get("/jobs/{job_id}")
async def get_job(
job_id: str,
authorization: str | None = Header(default=None),
) -> dict:
_require_api_key(authorization)
job = _get_job(job_id)
if job is None:
raise HTTPException(status_code=404, detail="job not found")
for k, v in job.items():
if hasattr(v, "isoformat"):
job[k] = v.isoformat()
return job
@app.get("/health")
async def health() -> dict:
missing = model_weights.missing_paths(WEIGHTS_DIR)
return {
"ok": True,
"pipeline": "longcat-avatar-1.5",
"model_loaded": not missing,
"missing_weights": [str(path) for path in missing],
"weights_dir": str(WEIGHTS_DIR),
"checkpoint_dir": str(CHECKPOINT_DIR),
"base_model_dir": str(BASE_MODEL_DIR),
"resolution": RESOLUTION,
"max_segments": MAX_SEGMENTS,
}