Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
"""
Avatar reconstruction service: selfie photos to a rigged GLB avatar.
Same external API contract as the original InstantMesh service, so the
three.ws backend (api/_providers/gcp.js, `reconstruct` mode) requires no
changes:
POST /reconstruct { images: [data-uri|https-url, ...], job_id?: str }
→ 202 { job_id, status: "queued" }
GET /jobs/:id → { job_id, status, glb_url?, error?, error_kind?, updated_at }
On failure, `error_kind` says whose problem it is:
"input" the photos were unusable; `error` is the
caller-facing reason ("no face detected …").
"internal" a service fault; `error` is an opaque
correlation id to join against the logs.
GET /health → { ok, pipeline, model_loaded, uv_map_ready, geometry_morph }
Environment variables (all required unless noted):
API_KEY — shared bearer secret (set via GCP Secret Manager)
GCS_BUCKET — Cloud Storage bucket for output GLBs
FIRESTORE_PROJECT — GCP project hosting Firestore
MAX_CONCURRENT_JOBS — parallel jobs (default: 2; CPU-only pipeline)
"""
from __future__ import annotations
import asyncio
import logging
import os
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from fastapi import FastAPI, HTTPException, Header, BackgroundTasks
from google.cloud import firestore, storage
from pydantic import BaseModel, Field
import face_pipeline
from worker_security import require_api_key, safe_error
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
log = logging.getLogger("main")
# ── config ─────────────────────────────────────────────────────────────────────
API_KEY = os.environ["API_KEY"]
GCS_BUCKET = os.environ["GCS_BUCKET"]
FIRESTORE_PROJECT = os.environ["FIRESTORE_PROJECT"]
MAX_CONCURRENT = int(os.environ.get("MAX_CONCURRENT_JOBS", "2"))
# ── global state ───────────────────────────────────────────────────────────────
_db: firestore.Client | None = None
_bucket: storage.Bucket | None = None
_job_sem: asyncio.Semaphore | None = None
# ── lifespan ───────────────────────────────────────────────────────────────────
@asynccontextmanager
async def lifespan(app: FastAPI):
global _db, _bucket, _job_sem
_job_sem = asyncio.Semaphore(MAX_CONCURRENT)
# Warm up the UV map and template GLBs on startup (loads them into memory).
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, face_pipeline._get_uv_map)
_db = firestore.Client(project=FIRESTORE_PROJECT)
_bucket = storage.Client().bucket(GCS_BUCKET)
log.info("Server ready — pipeline=face_texture_transfer, max_concurrent=%d", MAX_CONCURRENT)
yield
app = FastAPI(title="avatar-reconstruction", lifespan=lifespan)
# ── auth ───────────────────────────────────────────────────────────────────────
def _require_api_key(authorization: str) -> None:
try:
require_api_key(authorization, API_KEY)
except PermissionError as exc:
raise HTTPException(status_code=401, detail=str(exc)) from exc
# ── Firestore helpers ──────────────────────────────────────────────────────────
def _set_job(job_id: str, data: dict) -> None:
_db.collection("avatar_reconstruction_jobs").document(job_id).set(
{**data, "updated_at": datetime.now(timezone.utc)},
merge=True,
)
def _get_job(job_id: str) -> dict | None:
doc = _db.collection("avatar_reconstruction_jobs").document(job_id).get()
return doc.to_dict() if doc.exists else None
# ── job worker ─────────────────────────────────────────────────────────────────
async def _process_job(job_id: str, image_sources: list[str], body_type: str) -> None:
async with _job_sem:
_set_job(job_id, {"status": "running"})
loop = asyncio.get_event_loop()
try:
log.info("[%s] running pipeline", job_id)
glb_bytes = await loop.run_in_executor(
None,
lambda: face_pipeline.process(job_id, image_sources, body_type),
)
log.info("[%s] uploading %d bytes to GCS", job_id, len(glb_bytes))
blob_name = f"avatars/{job_id}.glb"
blob = _bucket.blob(blob_name)
await loop.run_in_executor(
None,
lambda: blob.upload_from_string(glb_bytes, content_type="model/gltf-binary"),
)
glb_url = f"https://storage.googleapis.com/{GCS_BUCKET}/{blob_name}"
_set_job(job_id, {"status": "done", "glb_url": glb_url})
log.info("[%s] done → %s", job_id, glb_url)
except face_pipeline.InputError as exc:
# The photos, not the service. Hand the reason back verbatim, it is
# the only thing that tells the user what to change, and the site's
# job-error mapping reads it (see face_pipeline.InputError). Logged at
# WARNING with no traceback so a rejected upload never looks like an
# outage in the error log.
log.warning("[%s] rejected input: %s", job_id, exc)
_set_job(job_id, {
"status": "failed",
"error": str(exc),
"error_kind": "input",
})
except Exception as exc:
_set_job(job_id, {
"status": "failed",
"error": safe_error(exc, context=f"[{job_id}] pipeline"),
"error_kind": "internal",
})
# ── routes ─────────────────────────────────────────────────────────────────────
class ReconstructRequest(BaseModel):
images: list[str] = Field(..., min_length=1, max_length=6)
job_id: str | None = None
body_type: str = "neutral" # 'male' | 'female' | 'neutral'
@app.post("/reconstruct", status_code=202)
async def reconstruct(
body: ReconstructRequest,
background_tasks: BackgroundTasks,
authorization: str = Header(...),
) -> dict:
_require_api_key(authorization)
job_id = body.job_id or str(uuid.uuid4())
_set_job(job_id, {
"job_id": job_id,
"status": "queued",
"image_count": len(body.images),
"body_type": body.body_type,
"created_at": datetime.now(timezone.utc),
})
background_tasks.add_task(_process_job, job_id, body.images, body.body_type)
return {"job_id": job_id, "status": "queued"}
@app.get("/jobs/{job_id}")
async def get_job(job_id: str, authorization: str = Header(...)) -> dict:
_require_api_key(authorization)
job = _get_job(job_id)
if job is None:
raise HTTPException(status_code=404, detail="job not found")
for k, v in job.items():
if hasattr(v, "isoformat"):
job[k] = v.isoformat()
return job
@app.get("/health")
async def health() -> dict:
uv_ready = (face_pipeline.HERE / "face_uv_map.json").exists()
geometry_ready = face_pipeline.GEOMETRY_MORPH_ENABLED and face_pipeline._get_face_map() is not None
return {
"ok": True,
"pipeline": "face_texture_transfer_v2" if geometry_ready else "face_texture_transfer_v1",
"model_loaded": uv_ready,
"uv_map_ready": uv_ready,
"geometry_morph": bool(geometry_ready),
}