Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
"""
Effectful stages of the garment forge: every function here talks to a real
service: Vertex AI image generation, the deployed mesh-generation workers
(model-hunyuan3d-21-rtx → model-hunyuan3d-21 → model-trellis), the deployed
auto-rigger (model-rig), and Cloud Storage. The pure geometry/manifest logic
lives in garment_glb.py so it stays unit-testable without any of this.
Auth model: one shared bearer secret (the platform's avatar-reconstruction
key) authenticates this worker's own API and its calls to the other workers.
Vertex AI and GCS use the service account Cloud Run runs as (ADC).
"""
from __future__ import annotations
import base64
import io
import json
import logging
import os
import time
from urllib.parse import urlparse
import google.auth
import google.auth.transport.requests
import httpx
from google.api_core.exceptions import NotFound, PreconditionFailed
from google.cloud import storage
from PIL import Image
log = logging.getLogger("garment-forge.pipeline")
VERTEX_MODEL = os.environ.get("VERTEX_IMAGEN_MODEL", "gemini-2.5-flash-image")
VERTEX_LOCATION = os.environ.get("VERTEX_IMAGEN_LOCATION", "global")
VERTEX_IMAGE_SIZE = os.environ.get("VERTEX_IMAGE_SIZE", "2K").upper()
# How many times to ask the image model for a reference photo before giving up.
# Counts only the "answered 200, drew nothing" case; transport failures are
# already retried inside _post_with_retry.
_IMAGE_ATTEMPTS = max(1, int(os.environ.get("VERTEX_IMAGE_ATTEMPTS", "3")))
MESH_POLL_S = float(os.environ.get("MESH_POLL_S", "5"))
MESH_TIMEOUT_S = float(os.environ.get("MESH_TIMEOUT_S", "1200"))
RIG_TIMEOUT_S = float(os.environ.get("RIG_TIMEOUT_S", "600"))
_credentials = None
def _access_token() -> str:
global _credentials
if _credentials is None:
_credentials, _ = google.auth.default(
scopes=["https://www.googleapis.com/auth/cloud-platform"])
if not _credentials.valid:
_credentials.refresh(google.auth.transport.requests.Request())
return _credentials.token
def _vertex_host(location: str) -> str:
return ("https://aiplatform.googleapis.com" if location == "global"
else f"https://{location}-aiplatform.googleapis.com")
# How each slot should be photographed so the generated mesh matches the
# reference body's A-pose and the slot box's fit axis. The ghost-mannequin
# framing gives the mesh generator a worn, volume-filled shape instead of a
# flat-lay it cannot infer depth from.
_SLOT_POSE_HINTS = {
"top": "shown worn in three-dimensional filled-out shape as if on an invisible "
"person standing upright in an A-pose, arms angled 45 degrees downward",
"outerwear": "shown worn in three-dimensional filled-out shape as if on an "
"invisible person standing upright in an A-pose, arms angled 45 "
"degrees downward",
"bottom": "shown worn in three-dimensional filled-out shape as if on invisible "
"legs standing upright, legs straight and slightly apart",
"footwear": "the pair side by side, both facing forward, soles on the ground",
"hair": "shown in three-dimensional shape as if on an invisible head, facing forward",
"headwear": "shown in three-dimensional shape as if worn on an invisible head, "
"facing forward",
"glasses": "shown front-on in three-dimensional shape as if worn on an invisible face",
"accessory": "shown in three-dimensional worn shape, facing forward",
}
def _post_with_retry(client: httpx.Client, url: str, *, json_body: dict,
headers: dict, attempts: int = 3) -> httpx.Response:
"""POST with bounded backoff on transient failures (429/5xx/network).
4xx other than 429 returns immediately: the caller handles semantics."""
last_exc: Exception | None = None
for attempt in range(attempts):
try:
res = client.post(url, json=json_body, headers=headers)
if res.status_code == 429 or res.status_code >= 500:
last_exc = RuntimeError(f"{url} returned {res.status_code}")
else:
return res
except httpx.HTTPError as exc:
last_exc = exc
time.sleep(2.0 * (attempt + 1))
raise RuntimeError(f"gave up after {attempts} attempts: {last_exc}")
def generate_reference_image(prompt: str, slot: str) -> tuple[bytes, str]:
"""Text prompt → ghost-mannequin product photo via the platform's live
Vertex AI image lane. Returns (png/jpeg bytes, model label)."""
pose = _SLOT_POSE_HINTS.get(slot, _SLOT_POSE_HINTS["accessory"])
full_prompt = (
f"Professional ghost mannequin product photograph of {prompt}, {pose}. "
"Front view, centered, the entire item visible with a small margin, "
"plain uniform light gray studio background, soft even lighting, "
"no person, no mannequin, no props, no text, no watermark."
)
project = os.environ["GOOGLE_CLOUD_PROJECT"]
endpoint = (
f"{_vertex_host(VERTEX_LOCATION)}/v1/projects/{project}/locations/"
f"{VERTEX_LOCATION}/publishers/google/models/{VERTEX_MODEL}:generateContent"
)
body = {
"contents": [{"role": "user", "parts": [{"text": full_prompt}]}],
"generationConfig": {
"responseModalities": ["IMAGE"],
"imageConfig": {"aspectRatio": "1:1", "imageSize": VERTEX_IMAGE_SIZE},
},
}
headers = {"authorization": f"Bearer {_access_token()}",
"content-type": "application/json"}
def _attempt() -> tuple[bytes | None, str | None, bool]:
"""One generation call.
Returns `(image_bytes, None, False)` on success, or
`(None, reason, prompt_blocked)` when the model answered 200 with no
image. `prompt_blocked` separates the two very different no-image
cases: a REFUSED PROMPT (deterministic — the safety classifier rejects
this wording every time) from an EMPTY DRAW (stochastic — the same
prompt usually succeeds on the next roll).
"""
with httpx.Client(timeout=120) as client:
res = _post_with_retry(client, endpoint, json_body=body, headers=headers)
if res.status_code == 400:
# Older image models reject the imageSize knob; retry without it.
del body["generationConfig"]["imageConfig"]["imageSize"]
res = _post_with_retry(client, endpoint, json_body=body, headers=headers)
res.raise_for_status()
data = res.json()
# A blocked PROMPT produces no candidates at all, so the reason lives in
# promptFeedback, not in candidates[0].finishReason. Reading only the
# latter reported every such block as a bare "no image data" with no
# reason attached, which is what made the 2026-07-29 batch failures
# look transient when they were not.
feedback = data.get("promptFeedback") or {}
block = feedback.get("blockReason")
if block:
detail = feedback.get("blockReasonMessage") or ""
return None, f"prompt blocked: {block}{f' ({detail})' if detail else ''}", True
candidate = (data.get("candidates") or [{}])[0]
parts = candidate.get("content", {}).get("parts", [])
img_part = next((p for p in parts if p.get("inlineData", {}).get("data")), None)
if not img_part:
return None, candidate.get("finishReason") or "no finishReason", False
return base64.b64decode(img_part["inlineData"]["data"]), None, False
# A 200 carrying no image is a DIFFERENT failure from the transport errors
# _post_with_retry covers: the request succeeded and the model simply
# declined to draw. Image generation is stochastic, so re-rolling the same
# prompt usually succeeds — and without a re-roll one such response
# permanently loses a job that has already been queued, paid for in GPU
# time downstream, and counted against the batch.
last_reason = None
for attempt in range(_IMAGE_ATTEMPTS):
image, last_reason, prompt_blocked = _attempt()
if image is not None:
if attempt:
log.info("reference image took %d attempt(s) (prior: %s)", attempt + 1, last_reason)
return image, f"vertex-ai/{VERTEX_MODEL}"
if prompt_blocked:
# Deterministic: the classifier rejects this wording, so every
# further roll costs a call and ~10s to be refused identically.
# Fail immediately with wording the caller can act on.
raise RuntimeError(
f"reference image refused for slot '{slot}': {last_reason}. "
"The safety classifier rejects this prompt wording; rephrase the "
"garment description and resubmit."
)
if attempt + 1 >= _IMAGE_ATTEMPTS:
break # budget spent; fall through to the raise below
log.warning("Vertex returned no image data (finishReason: %s); re-rolling "
"attempt %d/%d", last_reason, attempt + 2, _IMAGE_ATTEMPTS)
time.sleep(1.5 * (attempt + 1))
raise RuntimeError(f"Vertex returned no image data after {_IMAGE_ATTEMPTS} attempts "
f"(last finishReason: {last_reason})")
def _mesh_backend_label(url: str) -> str:
host = urlparse(url).hostname or url
if "trellis" in host:
return "trellis"
if "hunyuan3d" in host:
return "hunyuan3d-2.1"
return host
def _poll_task(client: httpx.Client, base_url: str, task_id: str, api_key: str,
timeout_s: float, result_field: str) -> str:
deadline = time.time() + timeout_s
headers = {"authorization": f"Bearer {api_key}"}
# A single flaky poll (worker restarting, LB hiccup, 502) must not kill a
# multi-minute GPU job; only a persistent failure streak is a real outage.
consecutive_errors = 0
while time.time() < deadline:
try:
res = client.get(f"{base_url}/tasks/{task_id}", headers=headers)
res.raise_for_status()
task = res.json()
except (httpx.HTTPError, ValueError) as exc:
consecutive_errors += 1
if consecutive_errors >= 6:
raise RuntimeError(
f"lost contact with {base_url} while polling {task_id}: {exc}"
) from exc
time.sleep(MESH_POLL_S)
continue
consecutive_errors = 0
status = task.get("status")
if status == "done":
url = task.get(result_field)
if not url:
raise RuntimeError(f"task {task_id} done but no {result_field}")
return url
if status == "failed":
raise RuntimeError(f"task {task_id} failed: {task.get('error')}")
time.sleep(MESH_POLL_S)
raise TimeoutError(f"task {task_id} on {base_url} exceeded {timeout_s:.0f}s")
def download_gcs_url(url: str) -> bytes:
"""Download an https://storage.googleapis.com/<bucket>/<name> object via
the storage client, so worker-to-worker transfers never depend on public
object ACLs."""
parsed = urlparse(url)
if parsed.hostname != "storage.googleapis.com":
raise ValueError(f"not a GCS URL: {url}")
bucket_name, _, blob_name = parsed.path.lstrip("/").partition("/")
return storage.Client().bucket(bucket_name).blob(blob_name).download_as_bytes()
def generate_mesh(image_bytes: bytes, mesh_urls: list[str], api_key: str,
tier: str = "high") -> tuple[bytes, str]:
"""Image → textured GLB through the deployed mesh-generation failover
chain. Every backend speaks the same /infer + /tasks contract."""
data_uri = "data:image/png;base64," + base64.b64encode(image_bytes).decode()
errors: list[str] = []
with httpx.Client(timeout=60) as client:
for url in mesh_urls:
label = _mesh_backend_label(url)
try:
res = client.post(
f"{url}/infer",
json={"images": [data_uri], "tier": tier},
headers={"authorization": f"Bearer {api_key}"},
)
res.raise_for_status()
task_id = res.json()["task_id"]
log.info("mesh task %s queued on %s", task_id, label)
result_url = _poll_task(client, url, task_id, api_key,
MESH_TIMEOUT_S, "result_gcs_url")
return download_gcs_url(result_url), label
except Exception as exc: # noqa: BLE001: try the next rung
log.warning("mesh backend %s failed: %s", label, exc)
errors.append(f"{label}: {exc}")
raise RuntimeError("every mesh backend failed: " + " | ".join(errors))
def rig_composite(composite_bytes: bytes, rig_url: str, api_key: str,
bucket: storage.Bucket, job_id: str) -> bytes:
"""Composite GLB → rigged GLB via the deployed model-rig worker. The
composite is staged to GCS because /rig takes a URL, not bytes."""
blob = bucket.blob(f"garments/tmp/{job_id}-composite.glb")
blob.upload_from_string(composite_bytes, content_type="model/gltf-binary")
composite_url = f"https://storage.googleapis.com/{bucket.name}/{blob.name}"
with httpx.Client(timeout=60) as client:
res = client.post(
f"{rig_url}/rig",
json={"mesh_gcs_url": composite_url, "blendshapes": False},
headers={"authorization": f"Bearer {api_key}"},
)
res.raise_for_status()
task_id = res.json()["task_id"]
log.info("rig task %s queued", task_id)
result_url = _poll_task(client, rig_url, task_id, api_key,
RIG_TIMEOUT_S, "rigged_gcs_url")
rigged = download_gcs_url(result_url)
blob.delete()
return rigged
def make_thumbnail(image_bytes: bytes, size: int = 512) -> bytes:
"""Reference image → square WEBP thumbnail for the catalog card."""
img = Image.open(io.BytesIO(image_bytes)).convert("RGB")
img.thumbnail((size, size), Image.LANCZOS)
out = io.BytesIO()
img.save(out, format="WEBP", quality=88)
return out.getvalue()
def _try_make_public(blob: storage.Blob) -> None:
"""Garment assets must be CORS/anonymously readable by the viewer. On a
fine-grained-ACL bucket this sets the object ACL; if the bucket enforces
uniform access this is a no-op handled at the bucket IAM level instead."""
try:
blob.make_public()
except Exception as exc: # noqa: BLE001: bucket-level policy takes over
log.warning("make_public failed for %s (bucket-level policy governs): %s",
blob.name, exc)
def publish(bucket: storage.Bucket, slot: str, garment_id: str, version: int,
glb_bytes: bytes, manifest: dict, thumb_bytes: bytes) -> dict:
"""Write the immutable v<n> directory and append to the catalog."""
prefix = f"garments/{slot}/{garment_id}/v{version}"
uploads = [
(f"{prefix}/garment.glb", glb_bytes, "model/gltf-binary"),
(f"{prefix}/manifest.json", json.dumps(manifest, indent=2).encode(),
"application/json"),
(f"{prefix}/thumb.webp", thumb_bytes, "image/webp"),
]
for name, data, ctype in uploads:
blob = bucket.blob(name)
blob.cache_control = "public, max-age=31536000, immutable"
blob.upload_from_string(data, content_type=ctype)
_try_make_public(blob)
append_to_catalog(bucket, manifest)
base = f"https://storage.googleapis.com/{bucket.name}/{prefix}"
return {
"glb_url": f"{base}/garment.glb",
"manifest_url": f"{base}/manifest.json",
"thumb_url": f"{base}/thumb.webp",
}
def append_to_catalog(bucket: storage.Bucket, manifest: dict,
attempts: int = 6) -> None:
"""Append one manifest to garments/catalog.json with an optimistic
generation-match loop, so concurrent publishes never lose entries."""
blob_name = "garments/catalog.json"
for attempt in range(attempts):
blob = bucket.blob(blob_name)
try:
blob.reload()
generation = blob.generation
catalog = json.loads(blob.download_as_bytes())
if not isinstance(catalog, list):
raise RuntimeError("garments/catalog.json is not a JSON array")
except NotFound:
generation = 0
catalog = []
catalog.append(manifest)
blob.cache_control = "public, max-age=60"
try:
blob.upload_from_string(
json.dumps(catalog, indent=2), content_type="application/json",
if_generation_match=generation,
)
_try_make_public(blob)
return
except PreconditionFailed:
log.info("catalog append raced (attempt %d), retrying", attempt + 1)
time.sleep(0.5 * (attempt + 1))
raise RuntimeError(f"catalog append lost {attempts} generation races")
def next_version(bucket: storage.Bucket, slot: str, garment_id: str) -> int:
"""1 + highest published version of slot/id in the catalog (spec: publish
a change as v<n+1>, never overwrite v<n>)."""
try:
catalog = json.loads(bucket.blob("garments/catalog.json").download_as_bytes())
except NotFound:
return 1
versions = [int(m.get("version", 0)) for m in catalog
if m.get("slot") == slot and m.get("id") == garment_id]
return max(versions, default=0) + 1