Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
"""
PartCrafter inference service: one image to a part-separated 3D asset
(wgsxm/PartCrafter, MIT; structured 3D generation that decodes several
individually editable part meshes from one RGB image in a single pass).
This is the Forge "Parts" lane (work order 070): where every other image
lane returns one undivided mesh, this worker returns a GLB whose nodes are
already named parts (`part_01`, `part_02`, ...), ready for the same Restyle
Studio part-select flow that workers/segment's post-hoc split feeds, and
sized by the caller's requested part count instead of a learned guess.
API contract (consumed by the Forge router, same task shape as the other
image-to-3D workers):
POST /infer { images: [data-uri|url, ...], num_parts: int (2-16),
seed?: int, job_id?: str }
-> 202 { task_id, status: "queued" }
PartCrafter conditions on ONE image, repeated once per requested part; when
several images are sent the first is used and the task record reports
views_used: 1.
GET /tasks/:id -> { task_id, status, result_gcs_url?, result_url?,
parts?, part_count?, error? }
GET /results/:id.glb -> generated GLB when OUTPUT_DIR local storage is active
GET /health -> { ok, model, gpu_available, load_error, load_attempts }.
503 with ok:false once the model load has spent its
retry budget, so a dead instance is drained.
GET / -> { service, model, ready, endpoints }. Unauthenticated
descriptor for the platform's warmth probe.
Weights: gs://three-ws-model-weights/partcrafter (workers/model-partcrafter/stage-assets.sh).
Environment variables (README.md carries the full table):
API_KEY shared bearer secret
GCS_BUCKET optional Cloud Storage bucket for output meshes
OUTPUT_DIR local task and result volume when GCS_BUCKET is unset
PARTCRAFTER_WEIGHTS_ROOT local checkpoint root entrypoint.sh stages into (default: /app/pretrained_weights)
MAX_CONCURRENT max parallel inferences (default: 1)
IMAGE_FETCH_TIMEOUT_S per-attempt image fetch timeout (default: 30)
MODEL_LOAD_ATTEMPTS model-load attempts before the error latches (default: 4)
MODEL_LOAD_RETRY_BASE_S / MODEL_LOAD_RETRY_CAP_S load-retry backoff bounds
"""
from __future__ import annotations
import asyncio
import base64
import gc
import json
import logging
import os
import time
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Optional
from urllib.parse import urlsplit
import httpx
import torch
from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Response
from fastapi.responses import FileResponse
from google.api_core.exceptions import NotFound
from google.cloud import storage
from pydantic import BaseModel, Field
from pipeline import MAX_NUM_PARTS, PartGenerationError, run_partcrafter
from storage_backend import LocalStorage
from worker_security import (
UnsafeUrlError,
fetch_remote_bytes,
require_api_key,
safe_error,
)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
log = logging.getLogger("model-partcrafter")
API_KEY = os.environ["API_KEY"]
GCS_BUCKET = os.environ.get("GCS_BUCKET", "").strip()
OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "/output")
PARTCRAFTER_WEIGHTS_ROOT = os.environ.get("PARTCRAFTER_WEIGHTS_ROOT", "/app/pretrained_weights")
PARTCRAFTER_DIR = os.path.join(PARTCRAFTER_WEIGHTS_ROOT, "PartCrafter")
RMBG_DIR = os.path.join(PARTCRAFTER_WEIGHTS_ROOT, "RMBG-1.4")
MAX_CONCURRENT = int(os.environ.get("MAX_CONCURRENT", "1"))
MODEL_LOAD_ATTEMPTS = int(os.environ.get("MODEL_LOAD_ATTEMPTS", "4"))
MODEL_LOAD_RETRY_BASE_S = float(os.environ.get("MODEL_LOAD_RETRY_BASE_S", "15"))
MODEL_LOAD_RETRY_CAP_S = float(os.environ.get("MODEL_LOAD_RETRY_CAP_S", "120"))
IMAGE_FETCH_TIMEOUT_S = float(os.environ.get("IMAGE_FETCH_TIMEOUT_S", "30"))
FETCH_ATTEMPTS = 3
_bucket: Optional[storage.Bucket] = None
_local_storage: Optional[LocalStorage] = None
_sem: Optional[asyncio.Semaphore] = None
_ready: Optional[asyncio.Event] = None
_load_error: Optional[str] = None
_load_attempts = 0
_pipeline_loaded = False
# In-memory cache only, same invariant as workers/model-trellis2: the durable
# source of truth is the tasks/{task_id}.json blob in GCS.
_tasks: dict[str, dict] = {}
_task_locks: dict[str, asyncio.Lock] = {}
def _load_pipeline_sync() -> None:
global _pipeline_loaded
from pipeline import load_pipeline
log.info("Loading PartCrafter pipeline: weights=%s rmbg=%s", PARTCRAFTER_DIR, RMBG_DIR)
load_pipeline(PARTCRAFTER_DIR, RMBG_DIR)
_pipeline_loaded = True
log.info("PartCrafter pipeline loaded")
async def _load_pipeline_bg() -> None:
"""Load the pipeline off the request path; retry with backoff before
latching a failure. See workers/model-trellis2's identical function for
why the first failure is never latched: a transient blip must not turn a
warm, minScale-held instance into a permanently dead one."""
global _load_error, _load_attempts
loop = asyncio.get_event_loop()
attempts = max(1, MODEL_LOAD_ATTEMPTS)
for attempt in range(1, attempts + 1):
_load_attempts = attempt
try:
await loop.run_in_executor(None, _load_pipeline_sync)
_load_error = None
_ready.set()
return
except Exception as exc: # noqa: BLE001 - surfaced via /health + task status
if attempt >= attempts:
_load_error = safe_error(exc, context="model load")
log.error("PartCrafter pipeline load FAILED after %d attempt(s): %s", attempt, exc)
return
delay = min(MODEL_LOAD_RETRY_BASE_S * (2 ** (attempt - 1)), MODEL_LOAD_RETRY_CAP_S)
log.warning(
"PartCrafter pipeline load attempt %d/%d failed (%s); retrying in %.0fs",
attempt, attempts, exc, delay,
)
await asyncio.sleep(delay)
@asynccontextmanager
async def lifespan(app: FastAPI):
global _bucket, _local_storage, _sem, _ready
if GCS_BUCKET:
_bucket = storage.Client().bucket(GCS_BUCKET)
_local_storage = None
else:
_bucket = None
_local_storage = LocalStorage(OUTPUT_DIR)
_sem = asyncio.Semaphore(MAX_CONCURRENT)
_ready = asyncio.Event()
asyncio.create_task(_load_pipeline_bg())
log.info("Service starting, pipeline loading in background (max_concurrent=%d)", MAX_CONCURRENT)
yield
app = FastAPI(title="model-partcrafter", lifespan=lifespan)
def _require_api_key(authorization: str) -> None:
try:
require_api_key(authorization, API_KEY)
except PermissionError as exc:
raise HTTPException(status_code=401, detail=str(exc)) from exc
class ImageSourceError(ValueError):
"""A caller-supplied image could not be read: see workers/model-trellis2."""
def _source_label(src: str) -> str:
if src.startswith("data:image"):
return "inline data uri"
host = urlsplit(src).hostname
return host or src[:60]
def _fetch_image_bytes(src: str) -> bytes:
label = _source_label(src)
last_exc: Optional[BaseException] = None
for attempt in range(1, FETCH_ATTEMPTS + 1):
try:
return fetch_remote_bytes(src, timeout=IMAGE_FETCH_TIMEOUT_S)
except UnsafeUrlError as exc:
raise ImageSourceError(f"refused to fetch image source ({label}): {exc}") from exc
except httpx.HTTPStatusError as exc:
raise ImageSourceError(
f"image source {label} returned HTTP {exc.response.status_code}"
) from exc
except httpx.HTTPError as exc:
last_exc = exc
if attempt < FETCH_ATTEMPTS:
log.warning(
"image fetch attempt %d/%d for %s failed (%s); retrying",
attempt, FETCH_ATTEMPTS, label, type(exc).__name__,
)
time.sleep(min(2 ** attempt, 5))
except ValueError as exc:
raise ImageSourceError(f"image source {label} rejected: {exc}") from exc
raise ImageSourceError(
f"image source {label} unreachable after {FETCH_ATTEMPTS} attempts "
f"({type(last_exc).__name__}); check the URL is publicly readable"
)
def _decode_image_bytes(src: str) -> bytes:
if src.startswith("data:image"):
b64 = src.split(",", 1)[1]
try:
return base64.b64decode(b64)
except Exception as exc: # noqa: BLE001 - caller's own payload
raise ImageSourceError(f"inline data uri is not valid base64: {exc}") from exc
if src.startswith("https://"):
return _fetch_image_bytes(src)
raise ImageSourceError(f"unsupported image source: {src[:60]}")
def _task_blob(task_id: str):
if _bucket is None:
raise RuntimeError("GCS task storage is not active")
return _bucket.blob(f"tasks/{task_id}.json")
def _using_gcs() -> bool:
return bool(GCS_BUCKET)
def _get_local_storage() -> LocalStorage:
global _local_storage
if _local_storage is None:
_local_storage = LocalStorage(OUTPUT_DIR)
return _local_storage
async def _update_task(task_id: str, **fields) -> dict:
lock = _task_locks.setdefault(task_id, asyncio.Lock())
async with lock:
task = _tasks.setdefault(task_id, {"task_id": task_id})
task.update(fields)
task["updated_at"] = time.time()
snapshot = task.copy()
loop = asyncio.get_event_loop()
if _using_gcs():
payload = json.dumps(snapshot)
await loop.run_in_executor(
None,
lambda: _task_blob(task_id).upload_from_string(
payload, content_type="application/json"
),
)
else:
await loop.run_in_executor(None, _get_local_storage().write_task, snapshot)
return task
_TERMINAL_STATUSES = frozenset({"done", "failed"})
async def _resolve_task(task_id: str) -> dict:
task = _tasks.get(task_id)
if task is not None and task.get("status") in _TERMINAL_STATUSES:
return task
loop = asyncio.get_event_loop()
try:
if _using_gcs():
data = await loop.run_in_executor(None, _task_blob(task_id).download_as_bytes)
task = json.loads(data)
else:
task = await loop.run_in_executor(None, _get_local_storage().read_task, task_id)
except (NotFound, FileNotFoundError, ValueError):
if task is not None:
return task
raise HTTPException(status_code=404, detail="task not found")
except Exception as exc:
raise HTTPException(status_code=502, detail=safe_error(exc, context="task lookup")) from exc
if task.get("status") in _TERMINAL_STATUSES:
_tasks[task_id] = task
return task
def _generate_sync(image_bytes: bytes, output_glb: Path, num_parts: int, seed: int) -> dict:
return run_partcrafter(
image_bytes,
output_glb,
partcrafter_weights_dir=PARTCRAFTER_DIR,
rmbg_weights_dir=RMBG_DIR,
num_parts=num_parts,
seed=seed,
)
def _release_gpu_memory() -> None:
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
async def _run_job(task_id: str, images: list[str], num_parts: int, seed: int | None) -> None:
if _load_error:
await _update_task(task_id, status="failed", error=f"pipeline unavailable: {_load_error}")
return
try:
await asyncio.wait_for(_ready.wait(), timeout=900)
except asyncio.TimeoutError:
await _update_task(task_id, status="failed", error="pipeline not ready (model load timed out)")
return
if _load_error:
await _update_task(task_id, status="failed", error=f"pipeline unavailable: {_load_error}")
return
async with _sem:
await _update_task(task_id, status="running")
loop = asyncio.get_event_loop()
t0 = time.time()
import tempfile
try:
image_bytes = await loop.run_in_executor(None, _decode_image_bytes, images[0])
with tempfile.TemporaryDirectory() as tmp:
output_glb = Path(tmp) / "parts.glb"
result = await loop.run_in_executor(
None, _generate_sync, image_bytes, output_glb, num_parts, seed if seed is not None else 0
)
glb_bytes = result["glb_bytes"]
if _using_gcs():
blob_name = f"raw-meshes/partcrafter/{task_id}.glb"
blob = _bucket.blob(blob_name)
await loop.run_in_executor(
None, lambda: blob.upload_from_string(glb_bytes, content_type="model/gltf-binary")
)
result_fields = {"result_gcs_url": f"https://storage.googleapis.com/{GCS_BUCKET}/{blob_name}"}
result_label = result_fields["result_gcs_url"]
else:
result_path = await loop.run_in_executor(
None, _get_local_storage().write_result, task_id, glb_bytes
)
result_fields = {"result_url": f"/results/{task_id}.glb", "result_path": str(result_path)}
result_label = str(result_path)
elapsed = time.time() - t0
await _update_task(
task_id,
status="done",
**result_fields,
parts=result["parts"],
part_count=result["part_count"],
views_used=1,
bytes=len(glb_bytes),
elapsed_ms=int(elapsed * 1000),
)
log.info(
"[%s] done in %.1fs (parts=%d) %d bytes -> %s",
task_id, elapsed, result["part_count"], len(glb_bytes), result_label,
)
except ImageSourceError as exc:
log.warning("[%s] image source rejected: %s", task_id, exc)
await _update_task(task_id, status="failed", error=str(exc), elapsed_ms=int((time.time() - t0) * 1000))
except PartGenerationError as exc:
log.error("[%s] PartCrafter produced no usable parts: %s", task_id, exc)
await _update_task(task_id, status="failed", error=str(exc), elapsed_ms=int((time.time() - t0) * 1000))
except Exception as exc:
await _update_task(
task_id, status="failed", error=safe_error(exc, context=f"[{task_id}] inference"),
elapsed_ms=int((time.time() - t0) * 1000),
)
finally:
await loop.run_in_executor(None, _release_gpu_memory)
class InferRequest(BaseModel):
images: list[str] = Field(..., min_length=1, max_length=6)
num_parts: int = Field(..., ge=2, le=MAX_NUM_PARTS)
job_id: str | None = None
seed: int | None = None
@app.post("/infer", status_code=202)
async def infer(body: InferRequest, background_tasks: BackgroundTasks, authorization: str = Header(...)) -> dict:
_require_api_key(authorization)
if _load_error:
raise HTTPException(status_code=503, detail=f"pipeline unavailable: {_load_error}")
task_id = str(uuid.uuid4())
await _update_task(task_id, status="queued", model="partcrafter")
background_tasks.add_task(_run_job, task_id, body.images, body.num_parts, body.seed)
return {"task_id": task_id, "status": "queued"}
@app.get("/tasks/{task_id}")
async def get_task(task_id: str, authorization: str = Header(...)) -> dict:
_require_api_key(authorization)
return await _resolve_task(task_id)
@app.get("/results/{task_id}.glb")
async def get_result(task_id: str, authorization: str = Header(...)):
_require_api_key(authorization)
if _using_gcs():
raise HTTPException(status_code=404, detail="local result storage is not active")
try:
path = _get_local_storage().result_path(task_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail="result not found") from exc
if not path.is_file():
raise HTTPException(status_code=404, detail="result not found")
return FileResponse(path, media_type="model/gltf-binary", filename=path.name)
@app.get("/")
async def root() -> dict:
return {
"service": "model-partcrafter",
"model": "partcrafter",
"ready": bool(_ready and _ready.is_set()),
"endpoints": ["POST /infer", "GET /tasks/{task_id}", "GET /results/{task_id}.glb", "GET /health"],
}
@app.get("/health")
async def health(response: Response) -> dict:
dead = _load_error is not None
if dead:
response.status_code = 503
return {
"ok": not dead,
"model": "partcrafter",
"gpu_available": torch.cuda.is_available(),
"gpu_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,
"pipeline_loaded": _pipeline_loaded,
"ready": bool(_ready and _ready.is_set()),
"load_error": _load_error,
"load_attempts": _load_attempts,
"output_backend": "gcs" if _using_gcs() else "local",
"output_dir": None if _using_gcs() else str(_get_local_storage().output_dir),
}