Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(workers): add AniGen worker, the rig-on-generation image to 3D lane

nirholasba76f81733d4
diff --git a/workers/model-anigen/Dockerfile b/workers/model-anigen/Dockerfile
new file mode 100644
index 000000000..bdee6d4dd
--- /dev/null
+++ b/workers/model-anigen/Dockerfile
@@ -0,0 +1,82 @@
+# AniGen (VAST-AI-Research, MIT, LICENSE read 2026-10-11): one reference
+# image to a rigged 3D asset (skeleton + skin weights predicted in the same
+# pass). Translates upstream's conda setup.sh into a reproducible pip build,
+# deliberately EXCLUDING one of its three CUDA extensions:
+#
+# * torch 2.4.0 + cu121, upstream setup.sh's own default matrix for
+# Python <=3.12.
+# * spconv-cu121 and pytorch3d (Meta, BSD, license read 2026-10-11):
+# installed, no restriction.
+# * nvdiffrast is NEVER installed. Its license (NVLabs/nvdiffrast,
+# "Nvidia Source Code License (1-Way Commercial)") bars commercial use,
+# which this worker's paid Forge lane cannot satisfy. patch_anigen.py
+# removes the two module-level imports that would otherwise pull it in
+# (see that file's docstring for the full nvdiffrast-usage trace); this
+# worker always runs the pipeline with fill_holes=False, texture_size=0
+# (pipeline.py), so nothing in its call path ever touches the renderer
+# that would need it.
+FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+ PYTHONUNBUFFERED=1 \
+ PIP_NO_CACHE_DIR=1 \
+ DEBIAN_FRONTEND=noninteractive \
+ PYTHONPATH=/app/anigen-src \
+ HF_HUB_OFFLINE=1 \
+ TRANSFORMERS_OFFLINE=1 \
+ HF_HUB_DISABLE_TELEMETRY=1 \
+ DO_NOT_TRACK=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ python3.10 python3.10-dev python3-pip \
+ git curl build-essential ninja-build \
+ libgl1 libglib2.0-0 libgomp1 \
+ && rm -rf /var/lib/apt/lists/* \
+ && ln -sf /usr/bin/python3.10 /usr/bin/python3 \
+ && ln -sf /usr/bin/python3.10 /usr/bin/python \
+ && pip install --upgrade pip setuptools wheel
+
+# 1. torch, its own cached layer (every extension below links against it).
+RUN pip install torch==2.4.0 torchvision==0.19.0 --index-url https://download.pytorch.org/whl/cu121
+
+# 2. AniGen, pinned to the commit read on 2026-10-11
+# (https://github.com/VAST-AI-Research/AniGen/commit/c49db3d6b466537a02ccf2286688903d77af7e4f).
+ARG ANIGEN_COMMIT=c49db3d6b466537a02ccf2286688903d77af7e4f
+RUN git clone https://github.com/VAST-AI-Research/AniGen.git /app/anigen-src \
+ && git -C /app/anigen-src checkout ${ANIGEN_COMMIT} \
+ && rm -rf /app/anigen-src/.git
+
+# 3. The build-time nvdiffrast-removal patch, BEFORE anything imports the tree.
+COPY patch_anigen.py test_patch_anigen.py /app/
+RUN python3 /app/test_patch_anigen.py && python3 /app/patch_anigen.py /app/anigen-src
+
+# 4. requirements.txt, minus torch/torchvision (installed above) and minus
+# nvdiffrast/pytorch3d (installed separately below; spconv needs its own
+# CUDA-major-keyed wheel, same as workers/rig-nonhuman).
+RUN pip install -r /app/anigen-src/requirements.txt \
+ && pip install spconv-cu120 \
+ && pip install "fvcore>=0.1.5" iopath \
+ && pip install --no-build-isolation \
+ "git+https://github.com/facebookresearch/[email protected]"
+
+WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install -r /app/requirements.txt
+
+COPY worker_security.py storage_backend.py pipeline.py main.py entrypoint.sh /app/
+COPY test_app_contract.py /app/
+RUN chmod +x /app/entrypoint.sh
+
+# Build-time guard: served-contract tests need no GPU and no checkpoints, so
+# they run here and the image does not ship if the patch, the import chain,
+# or request handling broke.
+RUN python3 /app/test_app_contract.py
+
+# Checkpoints are staged in GCS (stage-assets.sh) and mounted at /weights by
+# Cloud Run; entrypoint.sh copies them into ANIGEN_CKPT_ROOT/ckpts/anigen/...,
+# the exact literal relative path AniGen's own from_pretrained call uses
+# (example.py), so inference needs no Hugging Face egress at runtime.
+
+EXPOSE 8080
+
+CMD ["/app/entrypoint.sh"]
diff --git a/workers/model-anigen/README.md b/workers/model-anigen/README.md
new file mode 100644
index 000000000..65290be55
--- /dev/null
+++ b/workers/model-anigen/README.md
@@ -0,0 +1,131 @@
+# model-anigen: rigged-at-generation worker
+
+Every other Forge image-to-3D lane (TRELLIS.2, Pixal3D, Hunyuan3D, TripoSG,
+...) hands back a bare, unrigged mesh; animating it is a separate step
+through `workers/rig` or `workers/rig-nonhuman`. This worker, built on
+[AniGen](https://github.com/VAST-AI-Research/AniGen) (VAST-AI-Research, MIT,
+LICENSE read 2026-10-11), predicts the mesh, its joint skeleton, and its
+per-vertex skin weights in one pass, from one reference photo. The GLB it
+writes is already animatable: no second rigging call needed.
+
+## How it works
+
+`AnigenImageTo3DPipeline` (imported directly, in-process: AniGen is a real
+Python package, not a subprocess CLI) samples a sparse structure and a SLat
+latent from the input image, decodes a mesh and a predicted skeleton
+together, repairs the skeleton's parent hierarchy, transfers skin weights
+onto the (optionally simplified) output mesh, and writes one GLB carrying
+mesh + joints + parents + skin weights + predicted vertex colors
+(`anigen/utils/export_utils.py`'s `convert_to_glb_from_data`). See
+`pipeline.py` for the exact call.
+
+## nvdiffrast: never installed
+
+AniGen's raster-based texture bake and hole-fill stages import
+`nvdiffrast.torch`. nvdiffrast's own license (NVLabs/nvdiffrast, "Nvidia
+Source Code License (1-Way Commercial)", section 3.3) restricts use to
+non-commercial research/evaluation, which this worker's monetized Forge lane
+cannot satisfy. Rather than ship a restricted-license dependency, this
+worker:
+
+1. Never installs the `nvdiffrast` package (absent from the Dockerfile).
+2. Always calls the pipeline with `fill_holes=False, texture_size=0`
+ (`pipeline.py`), so the raster-dependent stages never run.
+3. Applies a build-time source patch (`patch_anigen.py`) that removes the two
+ module-level `import nvdiffrast.torch as dr` statements that would
+ otherwise make the vendored checkout fail to import at all. A repo-wide
+ search (`gh api search/code`, 2026-10-11) confirmed these are the ONLY two
+ files in the inference-relevant package that reference nvdiffrast; neither
+ patch touches nvdiffrast's own source, since it is never vendored.
+
+`patch_anigen.py`'s docstring and `test_patch_anigen.py` (unit tests against
+real excerpts of AniGen's own source) carry the full trace. `pytorch3d`
+(Meta, BSD, license read 2026-10-11) is installed with no restriction.
+
+## API
+
+```
+POST /infer { images, seed?, job_id? }
+ -> 202 { task_id, status: "queued" } (Bearer API_KEY)
+GET /tasks/:id -> { task_id, status, result_gcs_url?, result_url?,
+ rigged, joint_count?, error? }
+GET /results/:id.glb -> generated GLB (local OUTPUT_DIR mode only)
+GET /health -> { ok, model, gpu_available, gpu_name, load_error }
+```
+
+Same task shape as every other Forge image lane (`POST /infer` + poll
+`GET /tasks/:id`), so the Forge router dispatches to it exactly as it does
+TRELLIS.2 or Pixal3D. AniGen conditions on one image; when several are sent
+the first is used and the task record reports `views_used: 1`.
+
+## Deploy
+
+One-time checkpoint staging (under `gs://three-ws-model-weights/anigen/`):
+
+```bash
+bash workers/model-anigen/stage-assets.sh
+```
+
+Build + deploy (Cloud Build pins the `three-ws-build@` service account):
+
+```bash
+gcloud builds submit --config workers/model-anigen/cloudbuild.yaml \
+ --substitutions=SHORT_SHA=manual$(date +%s) .
+```
+
+Service: `model-anigen`, us-central1, 1x L4 (`--no-gpu-zonal-redundancy`),
+scale-to-zero by default against the shared per-region L4 quota in
+`docs/ops/gcp-credits-plan.md`.
+
+Then point the Forge API at it:
+
+```bash
+gcloud run services update three-ws-api --region us-central1 \
+ --update-env-vars MODEL_ANIGEN_URL=$(gcloud run services describe \
+ model-anigen --region us-central1 --format='value(status.url)')
+```
+
+`MODEL_ANIGEN_URL` is unset by default, so until that command runs, the
+`anigen` Forge backend stays inactive (same invariant as every other
+self-hosted image lane).
+
+## Local tests (no GPU needed)
+
+```bash
+cd workers/model-anigen
+python3 test_patch_anigen.py # nvdiffrast-removal patch, against real AniGen source
+```
+
+`test_app_contract.py` additionally proves the served contract, but it needs
+the patched AniGen checkout and its CPU-side dependencies on `PYTHONPATH`, so
+it runs inside the built image (`docker build -t model-anigen
+workers/model-anigen`, which runs it as a build-time gate; see the Dockerfile
+tail) rather than bare on a dev machine.
+
+## Licensing
+
+AniGen is MIT (LICENSE read 2026-10-11). pytorch3d is BSD. nvdiffrast is
+never installed; see the section above.
+
+## Environment
+
+| Var | Default | Meaning |
+|---|---|---|
+| `API_KEY` | (secret) | shared bearer secret (`avatar-reconstruction-key`) |
+| `GCS_BUCKET` | (optional) | output bucket for rigged GLBs; local `OUTPUT_DIR` storage when unset |
+| `OUTPUT_DIR` | `/output` | local task/result volume when `GCS_BUCKET` is unset |
+| `ANIGEN_CKPT_ROOT` | `/app` | root `entrypoint.sh` stages `ckpts/anigen/...` under |
+| `MAX_CONCURRENT` | `1` | parallel inference jobs per instance |
+| `WEIGHTS_ROOT` | `/weights/anigen` | staged checkpoints, read by `entrypoint.sh` |
+| `MODEL_LOAD_ATTEMPTS` / `MODEL_LOAD_RETRY_BASE_S` / `MODEL_LOAD_RETRY_CAP_S` | `4` / `15` / `120` | model-load retry budget |
+| `IMAGE_FETCH_TIMEOUT_S` | `30` | per-attempt image fetch timeout |
+
+## Failure modes
+
+| Reported `error` | Cause | Fix |
+|---|---|---|
+| `unsupported image source: ...` / `image source ... rejected` | bad or unreachable `images[0]` entry | fix the URL; rejected before touching the GPU |
+| `AniGen reported success but wrote no output GLB` | the pipeline ran but produced nothing | retry; if persistent, check the input image actually shows one coherent subject |
+| `AniGen produced a mesh with no predicted skeleton` | same as above, specifically an empty joint set | same as above |
+| `pipeline unavailable: ...` / `pipeline not ready` | checkpoint load failed or is still in progress | check `/health`'s `load_error`; retry once the instance reports ready |
+| `internal error (ref <id>)` | anything unexpected | grep the service logs for the correlation id |
diff --git a/workers/model-anigen/cloudbuild.yaml b/workers/model-anigen/cloudbuild.yaml
new file mode 100644
index 000000000..0f3a1f480
--- /dev/null
+++ b/workers/model-anigen/cloudbuild.yaml
@@ -0,0 +1,80 @@
+# Build + deploy the AniGen "rigged at generation" worker.
+# Manual submit (SHORT_SHA is only auto-set by triggers):
+# gcloud builds submit --config workers/model-anigen/cloudbuild.yaml \
+# --substitutions=SHORT_SHA=manual$(date +%s) .
+serviceAccount: projects/aerial-vehicle-466722-p5/serviceAccounts/[email protected]
+
+substitutions:
+ _REGION: us-central1
+ _RUN_SA: avatar-reconstruction-sa@aerial-vehicle-466722-p5.iam.gserviceaccount.com
+ _SERVICE: model-anigen
+ _REPO: model-anigen
+ _GCS_BUCKET: three-ws-avatar-reconstructions
+ _WEIGHTS_BUCKET: three-ws-model-weights
+ # Scale-to-zero by default, same reasoning as workers/model-trellis2: the
+ # us-central1 L4 grant is shared, and a warm floor here is a quota decision,
+ # not a default. Raise with `gcloud run services update model-anigen
+ # --min-instances=1` once the grant allows.
+ _MIN_INSTANCES: "0"
+ _MAX_INSTANCES: "2"
+
+steps:
+ - id: build
+ name: gcr.io/cloud-builders/docker
+ dir: workers/model-anigen
+ args:
+ - build
+ - --tag
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA
+ - --tag
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest
+ - --cache-from
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest
+ - .
+
+ - id: push
+ name: gcr.io/cloud-builders/docker
+ args:
+ - push
+ - --all-tags
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server
+
+ - id: deploy
+ name: gcr.io/google.com/cloudsdktool/cloud-sdk:slim
+ entrypoint: gcloud
+ args:
+ - run
+ - deploy
+ - ${_SERVICE}
+ - --image=${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA
+ - --region=${_REGION}
+ - --service-account=${_RUN_SA}
+ - --platform=managed
+ - --port=8080
+ - --gpu=1
+ - --gpu-type=nvidia-l4
+ - --no-gpu-zonal-redundancy
+ - --no-cpu-throttling
+ - --cpu=4
+ - --memory=16Gi
+ - --min-instances=${_MIN_INSTANCES}
+ - --max-instances=${_MAX_INSTANCES}
+ - --timeout=900
+ - --startup-probe=tcpSocket.port=8080,initialDelaySeconds=0,periodSeconds=60,timeoutSeconds=60,failureThreshold=15
+ - --cpu-boost
+ - --allow-unauthenticated
+ - --add-volume=name=weights,type=cloud-storage,bucket=${_WEIGHTS_BUCKET}
+ - --add-volume-mount=volume=weights,mount-path=/weights
+ - --set-env-vars=GCS_BUCKET=${_GCS_BUCKET},MAX_CONCURRENT=1,WEIGHTS_ROOT=/weights/anigen
+ - --set-secrets=API_KEY=avatar-reconstruction-key:latest
+ - --liveness-probe=httpGet.path=/health,initialDelaySeconds=60,periodSeconds=60,timeoutSeconds=10,failureThreshold=3
+
+images:
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA
+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest
+
+options:
+ machineType: E2_HIGHCPU_32
+ logging: CLOUD_LOGGING_ONLY
+
+timeout: 3600s
diff --git a/workers/model-anigen/entrypoint.sh b/workers/model-anigen/entrypoint.sh
new file mode 100644
index 000000000..9c0dd49fe
--- /dev/null
+++ b/workers/model-anigen/entrypoint.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+# Wire staged AniGen checkpoints into the literal relative path its own
+# from_pretrained call uses (example.py: ss_flow_path='ckpts/anigen/ss_flow_solo',
+# slat_flow_path='ckpts/anigen/slat_flow_auto'), then serve.
+#
+# Checkpoints are COPIED from the FUSE-mounted bucket to local disk rather than
+# symlinked, same reasoning as workers/rig and workers/rig-nonhuman:
+# torch.load over gcsfuse does slow random reads, and a sequential cp of a few
+# GB is fast and deterministic.
+set -eu
+
+WEIGHTS_ROOT="${WEIGHTS_ROOT:-/weights/anigen}"
+ANIGEN_CKPT_ROOT="${ANIGEN_CKPT_ROOT:-/app}"
+DST="$ANIGEN_CKPT_ROOT/ckpts/anigen"
+
+if [ ! -e "$WEIGHTS_ROOT/ckpts/anigen" ]; then
+ echo "fatal: expected staged checkpoints at $WEIGHTS_ROOT/ckpts/anigen (run workers/model-anigen/stage-assets.sh)" >&2
+ exit 1
+fi
+
+mkdir -p "$ANIGEN_CKPT_ROOT/ckpts"
+rm -rf "$DST"
+echo "staging $WEIGHTS_ROOT/ckpts/anigen -> $DST"
+cp -r "$WEIGHTS_ROOT/ckpts/anigen" "$DST"
+
+exec uvicorn main:app --host 0.0.0.0 --port 8080 --workers 1 --timeout-keep-alive 300
diff --git a/workers/model-anigen/main.py b/workers/model-anigen/main.py
new file mode 100644
index 000000000..f63422aae
--- /dev/null
+++ b/workers/model-anigen/main.py
@@ -0,0 +1,452 @@
+"""
+AniGen inference service: one reference image to a rigged 3D asset
+(VAST-AI-Research/AniGen, MIT license; image-to-3D with the skeleton and skin
+weights predicted in the same pass, not a separate rigging step).
+
+Unlike every other Forge image lane (TRELLIS.2, Pixal3D, Hunyuan3D, ...),
+this worker's output is already animatable: the GLB it writes carries joints,
+bone hierarchy, and per-vertex skin weights alongside the mesh. See
+pipeline.py for the in-process AniGen call and patch_anigen.py for why
+nvdiffrast is never installed (its license forbids commercial use; AniGen's
+own optional raster-texture-bake step that needs it is always disabled here).
+
+API contract (consumed by the Forge router, same task shape as the other
+image-to-3D workers):
+ POST /infer { images: [data-uri|url, ...], seed?: int, job_id?: str }
+ -> 202 { task_id, status: "queued" }
+
+ AniGen conditions on ONE image; when several are sent the first is used and
+ the task record reports views_used: 1.
+
+ GET /tasks/:id -> { task_id, status, result_gcs_url?, result_url?,
+ rigged: true, joint_count?, error? }
+
+ GET /results/:id.glb -> generated GLB when OUTPUT_DIR local storage is active
+
+ GET /health -> { ok, model, gpu_available, load_error, load_attempts }.
+ 503 with ok:false once the model load has spent its
+ retry budget, so a dead instance is drained.
+
+ GET / -> { service, model, ready, endpoints }. Unauthenticated
+ descriptor for the platform's warmth probe.
+
+Weights: gs://three-ws-model-weights/anigen (workers/model-anigen/stage-assets.sh).
+
+Environment variables (README.md carries the full table):
+ API_KEY shared bearer secret
+ GCS_BUCKET optional Cloud Storage bucket for output meshes
+ OUTPUT_DIR local task and result volume when GCS_BUCKET is unset
+ ANIGEN_CKPT_ROOT local checkpoint root entrypoint.sh stages into (default: /app)
+ MAX_CONCURRENT max parallel inferences (default: 1)
+ IMAGE_FETCH_TIMEOUT_S per-attempt image fetch timeout (default: 30)
+ MODEL_LOAD_ATTEMPTS model-load attempts before the error latches (default: 4)
+ MODEL_LOAD_RETRY_BASE_S / MODEL_LOAD_RETRY_CAP_S load-retry backoff bounds
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import gc
+import io
+import json
+import logging
+import os
+import time
+import uuid
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlsplit
+
+import httpx
+import torch
+from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Response
+from fastapi.responses import FileResponse
+from google.api_core.exceptions import NotFound
+from google.cloud import storage
+from PIL import Image
+from pydantic import BaseModel, Field
+
+from pipeline import RigAtGenerationError, run_anigen
+from storage_backend import LocalStorage
+from worker_security import (
+ UnsafeUrlError,
+ fetch_remote_bytes,
+ require_api_key,
+ safe_error,
+)
+
+logging.basicConfig(
+ level=logging.INFO,
+ format="%(asctime)s %(levelname)s %(name)s %(message)s",
+)
+log = logging.getLogger("model-anigen")
+
+API_KEY = os.environ["API_KEY"]
+GCS_BUCKET = os.environ.get("GCS_BUCKET", "").strip()
+OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "/output")
+ANIGEN_CKPT_ROOT = os.environ.get("ANIGEN_CKPT_ROOT", "/app")
+SS_FLOW_PATH = os.path.join(ANIGEN_CKPT_ROOT, "ckpts", "anigen", "ss_flow_solo")
+SLAT_FLOW_PATH = os.path.join(ANIGEN_CKPT_ROOT, "ckpts", "anigen", "slat_flow_auto")
+MAX_CONCURRENT = int(os.environ.get("MAX_CONCURRENT", "1"))
+MODEL_LOAD_ATTEMPTS = int(os.environ.get("MODEL_LOAD_ATTEMPTS", "4"))
+MODEL_LOAD_RETRY_BASE_S = float(os.environ.get("MODEL_LOAD_RETRY_BASE_S", "15"))
+MODEL_LOAD_RETRY_CAP_S = float(os.environ.get("MODEL_LOAD_RETRY_CAP_S", "120"))
+IMAGE_FETCH_TIMEOUT_S = float(os.environ.get("IMAGE_FETCH_TIMEOUT_S", "30"))
+FETCH_ATTEMPTS = 3
+
+_bucket: Optional[storage.Bucket] = None
+_local_storage: Optional[LocalStorage] = None
+_sem: Optional[asyncio.Semaphore] = None
+_ready: Optional[asyncio.Event] = None
+_load_error: Optional[str] = None
+_load_attempts = 0
+_pipeline_loaded = False
+# In-memory cache only, same invariant as workers/model-trellis2: the durable
+# source of truth is the tasks/{task_id}.json blob in GCS.
+_tasks: dict[str, dict] = {}
+_task_locks: dict[str, asyncio.Lock] = {}
+
+
+def _load_pipeline_sync() -> None:
+ global _pipeline_loaded
+ from pipeline import load_pipeline
+
+ log.info("Loading AniGen pipeline: ss=%s slat=%s", SS_FLOW_PATH, SLAT_FLOW_PATH)
+ load_pipeline(SS_FLOW_PATH, SLAT_FLOW_PATH)
+ _pipeline_loaded = True
+ log.info("AniGen pipeline loaded")
+
+
+async def _load_pipeline_bg() -> None:
+ """Load the pipeline off the request path; retry with backoff before
+ latching a failure. See workers/model-trellis2's identical function for
+ why the first failure is never latched: a transient blip must not turn a
+ warm, minScale-held instance into a permanently dead one."""
+ global _load_error, _load_attempts
+ loop = asyncio.get_event_loop()
+ attempts = max(1, MODEL_LOAD_ATTEMPTS)
+ for attempt in range(1, attempts + 1):
+ _load_attempts = attempt
+ try:
+ await loop.run_in_executor(None, _load_pipeline_sync)
+ _load_error = None
+ _ready.set()
+ return
+ except Exception as exc: # noqa: BLE001 - surfaced via /health + task status
+ if attempt >= attempts:
+ _load_error = safe_error(exc, context="model load")
+ log.error("AniGen pipeline load FAILED after %d attempt(s): %s", attempt, exc)
+ return
+ delay = min(MODEL_LOAD_RETRY_BASE_S * (2 ** (attempt - 1)), MODEL_LOAD_RETRY_CAP_S)
+ log.warning(
+ "AniGen pipeline load attempt %d/%d failed (%s); retrying in %.0fs",
+ attempt, attempts, exc, delay,
+ )
+ await asyncio.sleep(delay)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+ global _bucket, _local_storage, _sem, _ready
+ if GCS_BUCKET:
+ _bucket = storage.Client().bucket(GCS_BUCKET)
+ _local_storage = None
+ else:
+ _bucket = None
+ _local_storage = LocalStorage(OUTPUT_DIR)
+ _sem = asyncio.Semaphore(MAX_CONCURRENT)
+ _ready = asyncio.Event()
+ asyncio.create_task(_load_pipeline_bg())
+ log.info("Service starting, pipeline loading in background (max_concurrent=%d)", MAX_CONCURRENT)
+ yield
+
+
+app = FastAPI(title="model-anigen", lifespan=lifespan)
+
+
+def _require_api_key(authorization: str) -> None:
+ try:
+ require_api_key(authorization, API_KEY)
+ except PermissionError as exc:
+ raise HTTPException(status_code=401, detail=str(exc)) from exc
+
+
+class ImageSourceError(ValueError):
+ """A caller-supplied image could not be read: see workers/model-trellis2."""
+
+
+def _source_label(src: str) -> str:
+ if src.startswith("data:image"):
+ return "inline data uri"
+ host = urlsplit(src).hostname
+ return host or src[:60]
+
+
+def _fetch_image_bytes(src: str) -> bytes:
+ label = _source_label(src)
+ last_exc: Optional[BaseException] = None
+ for attempt in range(1, FETCH_ATTEMPTS + 1):
+ try:
+ return fetch_remote_bytes(src, timeout=IMAGE_FETCH_TIMEOUT_S)
+ except UnsafeUrlError as exc:
+ raise ImageSourceError(f"refused to fetch image source ({label}): {exc}") from exc
+ except httpx.HTTPStatusError as exc:
+ raise ImageSourceError(
+ f"image source {label} returned HTTP {exc.response.status_code}"
+ ) from exc
+ except httpx.HTTPError as exc:
+ last_exc = exc
+ if attempt < FETCH_ATTEMPTS:
+ log.warning(
+ "image fetch attempt %d/%d for %s failed (%s); retrying",
+ attempt, FETCH_ATTEMPTS, label, type(exc).__name__,
+ )
+ time.sleep(min(2 ** attempt, 5))
+ except ValueError as exc:
+ raise ImageSourceError(f"image source {label} rejected: {exc}") from exc
+ raise ImageSourceError(
+ f"image source {label} unreachable after {FETCH_ATTEMPTS} attempts "
+ f"({type(last_exc).__name__}); check the URL is publicly readable"
+ )
+
+
+def _decode_image_bytes(src: str) -> bytes:
+ if src.startswith("data:image"):
+ b64 = src.split(",", 1)[1]
+ try:
+ return base64.b64decode(b64)
+ except Exception as exc: # noqa: BLE001 - caller's own payload
+ raise ImageSourceError(f"inline data uri is not valid base64: {exc}") from exc
+ if src.startswith("https://"):
+ return _fetch_image_bytes(src)
+ raise ImageSourceError(f"unsupported image source: {src[:60]}")
+
+
+def _task_blob(task_id: str):
+ if _bucket is None:
+ raise RuntimeError("GCS task storage is not active")
+ return _bucket.blob(f"tasks/{task_id}.json")
+
+
+def _using_gcs() -> bool:
+ return bool(GCS_BUCKET)
+
+
+def _get_local_storage() -> LocalStorage:
+ global _local_storage
+ if _local_storage is None:
+ _local_storage = LocalStorage(OUTPUT_DIR)
+ return _local_storage
+
+
+async def _update_task(task_id: str, **fields) -> dict:
+ lock = _task_locks.setdefault(task_id, asyncio.Lock())
+ async with lock:
+ task = _tasks.setdefault(task_id, {"task_id": task_id})
+ task.update(fields)
+ task["updated_at"] = time.time()
+ snapshot = task.copy()
+ loop = asyncio.get_event_loop()
+ if _using_gcs():
+ payload = json.dumps(snapshot)
+ await loop.run_in_executor(
+ None,
+ lambda: _task_blob(task_id).upload_from_string(
+ payload, content_type="application/json"
+ ),
+ )
+ else:
+ await loop.run_in_executor(None, _get_local_storage().write_task, snapshot)
+ return task
+
+
+_TERMINAL_STATUSES = frozenset({"done", "failed"})
+
+
+async def _resolve_task(task_id: str) -> dict:
+ task = _tasks.get(task_id)
+ if task is not None and task.get("status") in _TERMINAL_STATUSES:
+ return task
+ loop = asyncio.get_event_loop()
+ try:
+ if _using_gcs():
+ data = await loop.run_in_executor(None, _task_blob(task_id).download_as_bytes)
+ task = json.loads(data)
+ else:
+ task = await loop.run_in_executor(None, _get_local_storage().read_task, task_id)
+ except (NotFound, FileNotFoundError, ValueError):
+ if task is not None:
+ return task
+ raise HTTPException(status_code=404, detail="task not found")
+ except Exception as exc:
+ raise HTTPException(status_code=502, detail=safe_error(exc, context="task lookup")) from exc
+ if task.get("status") in _TERMINAL_STATUSES:
+ _tasks[task_id] = task
+ return task
+
+
+def _rig_sync(image_bytes: bytes, output_glb: Path, seed: int) -> dict:
+ run_anigen(
+ image_bytes,
+ output_glb,
+ ss_flow_path=SS_FLOW_PATH,
+ slat_flow_path=SLAT_FLOW_PATH,
+ seed=seed,
+ )
+ import trimesh
+
+ mesh = trimesh.load(output_glb, file_type="glb", force="scene")
+ joint_count = 0
+ for geom in mesh.geometry.values() if hasattr(mesh, "geometry") else []:
+ skeleton = getattr(geom, "skeleton", None)
+ if skeleton is not None:
+ joint_count = max(joint_count, len(getattr(skeleton, "joints", []) or []))
+ return {"glb_bytes": output_glb.read_bytes(), "joint_count": joint_count}
+
+
+def _release_gpu_memory() -> None:
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ torch.cuda.ipc_collect()
+
+
+async def _run_job(task_id: str, images: list[str], seed: int | None) -> None:
+ if _load_error:
+ await _update_task(task_id, status="failed", error=f"pipeline unavailable: {_load_error}")
+ return
+ try:
+ await asyncio.wait_for(_ready.wait(), timeout=900)
+ except asyncio.TimeoutError:
+ await _update_task(task_id, status="failed", error="pipeline not ready (model load timed out)")
+ return
+ if _load_error:
+ await _update_task(task_id, status="failed", error=f"pipeline unavailable: {_load_error}")
+ return
+
+ async with _sem:
+ await _update_task(task_id, status="running")
+ loop = asyncio.get_event_loop()
+ t0 = time.time()
+ import tempfile
+
+ try:
+ image_bytes = await loop.run_in_executor(None, _decode_image_bytes, images[0])
+ with tempfile.TemporaryDirectory() as tmp:
+ output_glb = Path(tmp) / "rigged.glb"
+ result = await loop.run_in_executor(
+ None, _rig_sync, image_bytes, output_glb, seed if seed is not None else 42
+ )
+ glb_bytes = result["glb_bytes"]
+
+ if _using_gcs():
+ blob_name = f"raw-meshes/anigen/{task_id}.glb"
+ blob = _bucket.blob(blob_name)
+ await loop.run_in_executor(
+ None, lambda: blob.upload_from_string(glb_bytes, content_type="model/gltf-binary")
+ )
+ result_fields = {"result_gcs_url": f"https://storage.googleapis.com/{GCS_BUCKET}/{blob_name}"}
+ result_label = result_fields["result_gcs_url"]
+ else:
+ result_path = await loop.run_in_executor(
+ None, _get_local_storage().write_result, task_id, glb_bytes
+ )
+ result_fields = {"result_url": f"/results/{task_id}.glb", "result_path": str(result_path)}
+ result_label = str(result_path)
+
+ elapsed = time.time() - t0
+ await _update_task(
+ task_id,
+ status="done",
+ **result_fields,
+ rigged=True,
+ joint_count=result["joint_count"],
+ views_used=1,
+ bytes=len(glb_bytes),
+ elapsed_ms=int(elapsed * 1000),
+ )
+ log.info(
+ "[%s] done in %.1fs (joints=%d) %d bytes -> %s",
+ task_id, elapsed, result["joint_count"], len(glb_bytes), result_label,
+ )
+ except ImageSourceError as exc:
+ log.warning("[%s] image source rejected: %s", task_id, exc)
+ await _update_task(task_id, status="failed", error=str(exc), elapsed_ms=int((time.time() - t0) * 1000))
+ except RigAtGenerationError as exc:
+ log.error("[%s] AniGen produced no usable rig: %s", task_id, exc)
+ await _update_task(task_id, status="failed", error=str(exc), elapsed_ms=int((time.time() - t0) * 1000))
+ except Exception as exc:
+ await _update_task(
+ task_id, status="failed", error=safe_error(exc, context=f"[{task_id}] inference"),
+ elapsed_ms=int((time.time() - t0) * 1000),
+ )
+ finally:
+ await loop.run_in_executor(None, _release_gpu_memory)
+
+
+class InferRequest(BaseModel):
+ images: list[str] = Field(..., min_length=1, max_length=6)
+ job_id: str | None = None
+ seed: int | None = None
+
+
[email protected]("/infer", status_code=202)
+async def infer(body: InferRequest, background_tasks: BackgroundTasks, authorization: str = Header(...)) -> dict:
+ _require_api_key(authorization)
+ if _load_error:
+ raise HTTPException(status_code=503, detail=f"pipeline unavailable: {_load_error}")
+ task_id = str(uuid.uuid4())
+ await _update_task(task_id, status="queued", model="anigen")
+ background_tasks.add_task(_run_job, task_id, body.images, body.seed)
+ return {"task_id": task_id, "status": "queued"}
+
+
[email protected]("/tasks/{task_id}")
+async def get_task(task_id: str, authorization: str = Header(...)) -> dict:
+ _require_api_key(authorization)
+ return await _resolve_task(task_id)
+
+
[email protected]("/results/{task_id}.glb")
+async def get_result(task_id: str, authorization: str = Header(...)):
+ _require_api_key(authorization)
+ if _using_gcs():
+ raise HTTPException(status_code=404, detail="local result storage is not active")
+ try:
+ path = _get_local_storage().result_path(task_id)
+ except ValueError as exc:
+ raise HTTPException(status_code=404, detail="result not found") from exc
+ if not path.is_file():
+ raise HTTPException(status_code=404, detail="result not found")
+ return FileResponse(path, media_type="model/gltf-binary", filename=path.name)
+
+
+async def root() -> dict:
+ return {
+ "service": "model-anigen",
+ "model": "anigen",
+ "ready": bool(_ready and _ready.is_set()),
+ "endpoints": ["POST /infer", "GET /tasks/{task_id}", "GET /results/{task_id}.glb", "GET /health"],
+ }
+
+
+async def health(response: Response) -> dict:
+ dead = _load_error is not None
+ if dead:
+ response.status_code = 503
+ return {
+ "ok": not dead,
+ "model": "anigen",
+ "gpu_available": torch.cuda.is_available(),
+ "gpu_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,
+ "pipeline_loaded": _pipeline_loaded,
+ "ready": bool(_ready and _ready.is_set()),
+ "load_error": _load_error,
+ "load_attempts": _load_attempts,
+ "output_backend": "gcs" if _using_gcs() else "local",
+ "output_dir": None if _using_gcs() else str(_get_local_storage().output_dir),
+ }
diff --git a/workers/model-anigen/patch_anigen.py b/workers/model-anigen/patch_anigen.py
new file mode 100644
index 000000000..6a98045ec
--- /dev/null
+++ b/workers/model-anigen/patch_anigen.py
@@ -0,0 +1,106 @@
+"""
+Build-time patch: remove every module-level dependency on nvdiffrast from the
+vendored AniGen checkout.
+
+Why: AniGen (VAST-AI-Research/AniGen) is MIT, but two of its files import
+nvdiffrast at module scope. nvdiffrast's own license (NVlabs/nvdiffrast,
+LICENSE.txt, "Nvidia Source Code License (1-Way Commercial)", section 3.3)
+restricts use to "non-commercially... for research or evaluation purposes
+only", which three.ws's monetized Forge lane cannot satisfy. Rather than
+install a restricted-license package, this worker never installs nvdiffrast
+at all and patches the two files that reference it so the package loads
+cleanly.
+
+Repo-wide search confirmed (gh api search/code, 2026-10-11) these are the
+ONLY two Python files under `anigen/` (excluding the optional, unused
+`anigen/4drecon/` video add-on) that reference nvdiffrast:
+
+ 1. anigen/utils/postprocessing_utils.py: `import nvdiffrast.torch as dr` at
+ module top. Used only inside `_fill_holes`, `bake_vertex_colors_to_texture`,
+ `bake_texture`, `render_multiview_mesh_colors`, `to_glb`, `simplify_gs` -
+ none of which this worker ever calls (it always runs the pipeline with
+ fill_holes=False, texture_size=0; see pipeline.py). Deleting the
+ module-level import is enough: Python resolves `dr.*` lazily inside
+ those function bodies, so the module still imports cleanly, and only an
+ accidental call to one of those unused functions would raise NameError.
+ 2. anigen/renderers/mesh_renderer.py: same import, consumed only by
+ MeshRenderer.__init__/.render. Moved into __init__ as a local import (the
+ lazy-import pattern anigen/renderers/__init__.py's own __getattr__
+ already uses for submodule loading) so merely importing mesh_renderer.py
+ (forced transitively: postprocessing_utils.py -> render_utils.py's
+ `from ..renderers import ... MeshRenderer` -> the package's lazy
+ __getattr__ -> mesh_renderer.py) no longer touches nvdiffrast.
+ Instantiating MeshRenderer still would, but nothing in this worker's
+ call path does.
+
+Each patch asserts its exact target text is present before editing, so a
+future AniGen version that moves or rewrites these lines fails the Docker
+build loudly instead of silently shipping unpatched (and therefore either
+broken, or secretly dependent on a restricted-license package).
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+POSTPROCESSING_UTILS_IMPORT = "import nvdiffrast.torch as dr\n"
+
+MESH_RENDERER_MODULE_IMPORT = "import nvdiffrast.torch as dr\n"
+MESH_RENDERER_INIT_MARKER = " def __init__(self, rendering_options={}, device='cuda'):\n"
+MESH_RENDERER_LOCAL_IMPORT = " import nvdiffrast.torch as dr\n"
+
+
+def patch_postprocessing_utils(path: Path) -> None:
+ text = path.read_text()
+ if POSTPROCESSING_UTILS_IMPORT not in text:
+ raise AssertionError(
+ f"{path}: expected top-level '{POSTPROCESSING_UTILS_IMPORT.strip()}' "
+ "not found; AniGen's postprocessing_utils.py has changed, re-check "
+ "the nvdiffrast surface before patching blind"
+ )
+ patched = text.replace(POSTPROCESSING_UTILS_IMPORT, "", 1)
+ if "nvdiffrast" in patched:
+ raise AssertionError(
+ f"{path}: nvdiffrast is still referenced after removing the import; "
+ "a new module-level usage appeared, re-check before patching blind"
+ )
+ path.write_text(patched)
+ print(f"patched {path}: removed module-level nvdiffrast import")
+
+
+def patch_mesh_renderer(path: Path) -> None:
+ text = path.read_text()
+ if MESH_RENDERER_MODULE_IMPORT not in text:
+ raise AssertionError(
+ f"{path}: expected top-level '{MESH_RENDERER_MODULE_IMPORT.strip()}' "
+ "not found; AniGen's mesh_renderer.py has changed, re-check the "
+ "nvdiffrast surface before patching blind"
+ )
+ if MESH_RENDERER_INIT_MARKER not in text:
+ raise AssertionError(
+ f"{path}: expected MeshRenderer.__init__ signature not found; "
+ "cannot safely relocate the nvdiffrast import"
+ )
+ without_top_import = text.replace(MESH_RENDERER_MODULE_IMPORT, "", 1)
+ patched = without_top_import.replace(
+ MESH_RENDERER_INIT_MARKER,
+ MESH_RENDERER_INIT_MARKER + MESH_RENDERER_LOCAL_IMPORT,
+ 1,
+ )
+ if "import nvdiffrast" not in patched:
+ raise AssertionError(f"{path}: patch lost the nvdiffrast import entirely")
+ path.write_text(patched)
+ print(f"patched {path}: moved nvdiffrast import into MeshRenderer.__init__")
+
+
+def main() -> None:
+ if len(sys.argv) != 2:
+ print("usage: patch_anigen.py <anigen-checkout-root>", file=sys.stderr)
+ sys.exit(2)
+ root = Path(sys.argv[1])
+ patch_postprocessing_utils(root / "anigen" / "utils" / "postprocessing_utils.py")
+ patch_mesh_renderer(root / "anigen" / "renderers" / "mesh_renderer.py")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/workers/model-anigen/pipeline.py b/workers/model-anigen/pipeline.py
new file mode 100644
index 000000000..611a2b775
--- /dev/null
+++ b/workers/model-anigen/pipeline.py
@@ -0,0 +1,93 @@
+"""
+AniGen (VAST-AI-Research/AniGen, MIT) in-process pipeline wrapper: a single
+reference image in, one rigged GLB out. Unlike workers/rig-nonhuman's UniRig
++ SkinTokens chain, AniGen is an importable Python pipeline (confirmed by its
+own minimal inference script, example.py), not a subprocess CLI, so this
+worker loads the model once and calls it directly.
+
+The pipeline always runs with fill_holes=False and texture_size=0: see
+patch_anigen.py's module docstring for why (nvdiffrast's restrictive
+license). This drops two things AniGen can optionally add, neither load-bearing
+for a rigged asset: raster-visibility hole-filling, and a baked UV texture.
+What ships instead: the real per-vertex colors AniGen's own decoder predicts
+(convert_to_glb_from_data's vertex_colors path), the real predicted skeleton,
+and the real predicted skin weights, every bit of it the model's genuine
+output.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+from pathlib import Path
+
+from PIL import Image
+
+log = logging.getLogger("model-anigen.pipeline")
+
+_lock = threading.Lock()
+_pipeline = None
+
+
+def load_pipeline(ss_flow_path: str, slat_flow_path: str, device: str = "cuda"):
+ """Load (once) and return the AnigenImageTo3DPipeline singleton.
+
+ Import is deferred to call time, not module top: this lets
+ test_pipeline.py exercise request validation and output handling without
+ AniGen's checkout or its GPU dependencies installed.
+ """
+ global _pipeline
+ with _lock:
+ if _pipeline is None:
+ from anigen.pipelines import AnigenImageTo3DPipeline
+
+ log.info("loading AniGen pipeline: ss=%s slat=%s", ss_flow_path, slat_flow_path)
+ pipeline = AnigenImageTo3DPipeline.from_pretrained(
+ ss_flow_path=ss_flow_path,
+ slat_flow_path=slat_flow_path,
+ device=device,
+ )
+ pipeline.cuda()
+ _pipeline = pipeline
+ return _pipeline
+
+
+class RigAtGenerationError(RuntimeError):
+ """AniGen ran but produced nothing usable: an empty mesh, or no output
+ file despite reporting success. Not the caller's fault; the caller
+ already passed a readable image."""
+
+
+def run_anigen(
+ image_bytes: bytes,
+ output_glb: Path,
+ *,
+ ss_flow_path: str,
+ slat_flow_path: str,
+ seed: int = 42,
+ joints_density: int = 1,
+) -> None:
+ """Run AniGen end to end on one reference image, writing a rigged GLB
+ (skeleton + skin weights + vertex colors) to `output_glb`."""
+ import io
+
+ pipeline = load_pipeline(ss_flow_path, slat_flow_path)
+ image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
+
+ outputs = pipeline.run(
+ image,
+ seed=seed,
+ joints_density=joints_density,
+ fill_holes=False,
+ texture_size=0,
+ output_glb=str(output_glb),
+ )
+
+ if not output_glb.exists():
+ raise RigAtGenerationError(
+ "AniGen reported success but wrote no output GLB"
+ )
+ joints = outputs.get("joints")
+ if joints is None or len(joints) == 0:
+ raise RigAtGenerationError(
+ "AniGen produced a mesh with no predicted skeleton"
+ )
diff --git a/workers/model-anigen/requirements.txt b/workers/model-anigen/requirements.txt
new file mode 100644
index 000000000..eae7b83ef
--- /dev/null
+++ b/workers/model-anigen/requirements.txt
@@ -0,0 +1,12 @@
+# This worker's own FastAPI/GCS layer. torch/torchvision, AniGen's own
+# requirements.txt, spconv, and pytorch3d install in their own Dockerfile
+# layers BEFORE this file (AniGen ships no pip-installable package; it is
+# imported straight from its vendored checkout on PYTHONPATH).
+trimesh>=4.4.0,<5
+
+fastapi>=0.115,<1
+uvicorn[standard]>=0.32,<1
+pydantic>=2.0.0
+httpx>=0.27.0
+
+google-cloud-storage==2.18.0
diff --git a/workers/model-anigen/stage-assets.sh b/workers/model-anigen/stage-assets.sh
new file mode 100644
index 000000000..32f2524d2
--- /dev/null
+++ b/workers/model-anigen/stage-assets.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Stage AniGen's checkpoints into the model-weights bucket. Run once (from any
+# machine with gcloud + python3 + huggingface_hub) before the first deploy,
+# and again only when bumping ANIGEN_COMMIT in the Dockerfile.
+#
+# bash workers/model-anigen/stage-assets.sh
+#
+# Layout produced under gs://three-ws-model-weights/anigen/:
+# ckpts/anigen/ss_flow_solo/... sparse-structure flow checkpoint
+# ckpts/anigen/slat_flow_auto/... SLat flow checkpoint
+#
+# This is the exact same call AniGen's own anigen/utils/ckpt_utils.py
+# (ensure_ckpts) makes at runtime when ckpts/ is missing; staging it here
+# ahead of time means entrypoint.sh never needs Hugging Face egress in
+# production.
+set -euo pipefail
+
+BUCKET="${BUCKET:-gs://three-ws-model-weights/anigen}"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+echo "==> AniGen checkpoints (HF: VAST-AI/AniGen)"
+python3 -c "
+from huggingface_hub import snapshot_download
+path = snapshot_download(
+ repo_id='VAST-AI/AniGen',
+ allow_patterns=['ckpts/**'],
+ local_dir='$WORK',
+)
+print('downloaded to', path)
+"
+
+echo "==> Upload to $BUCKET"
+gcloud storage cp -r "$WORK/ckpts" "$BUCKET/ckpts"
+
+echo "==> Staged. Verify:"
+gcloud storage ls -r "$BUCKET" | sed 's/^/ /'
diff --git a/workers/model-anigen/storage_backend.py b/workers/model-anigen/storage_backend.py
new file mode 100644
index 000000000..fcf7fc1a1
--- /dev/null
+++ b/workers/model-anigen/storage_backend.py
@@ -0,0 +1,63 @@
+"""Storage primitives for the standalone TRELLIS container.
+
+Production persists tasks and results in GCS because requests can move between
+Cloud Run instances. An NGC user runs one container with a mounted output
+volume, so the same service needs a local backend with atomic writes and strict
+task-id validation.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import uuid
+from pathlib import Path
+
+
+def normalize_task_id(task_id: str) -> str:
+ """Return a canonical UUID or reject a value that could escape the volume."""
+ parsed = uuid.UUID(task_id)
+ canonical = str(parsed)
+ if task_id.lower() != canonical:
+ raise ValueError("task_id must be a canonical UUID")
+ return canonical
+
+
+class LocalStorage:
+ """Durable task metadata and GLB results on a mounted local volume."""
+
+ def __init__(self, output_dir: str | os.PathLike[str]):
+ self.output_dir = Path(output_dir).expanduser().resolve()
+ self.tasks_dir = self.output_dir / "tasks"
+ self.results_dir = self.output_dir / "results"
+ self.tasks_dir.mkdir(parents=True, exist_ok=True)
+ self.results_dir.mkdir(parents=True, exist_ok=True)
+
+ def task_path(self, task_id: str) -> Path:
+ return self.tasks_dir / f"{normalize_task_id(task_id)}.json"
+
+ def result_path(self, task_id: str) -> Path:
+ return self.results_dir / f"{normalize_task_id(task_id)}.glb"
+
+ @staticmethod
+ def _atomic_write(path: Path, data: bytes) -> None:
+ temporary = path.with_suffix(path.suffix + ".tmp")
+ with temporary.open("wb") as handle:
+ handle.write(data)
+ handle.flush()
+ os.fsync(handle.fileno())
+ temporary.replace(path)
+
+ def write_task(self, task: dict) -> Path:
+ path = self.task_path(str(task["task_id"]))
+ payload = json.dumps(task, separators=(",", ":"), sort_keys=True).encode("utf-8")
+ self._atomic_write(path, payload)
+ return path
+
+ def read_task(self, task_id: str) -> dict:
+ return json.loads(self.task_path(task_id).read_text(encoding="utf-8"))
+
+ def write_result(self, task_id: str, glb_bytes: bytes) -> Path:
+ path = self.result_path(task_id)
+ self._atomic_write(path, glb_bytes)
+ return path
diff --git a/workers/model-anigen/test_app_contract.py b/workers/model-anigen/test_app_contract.py
new file mode 100644
index 000000000..2a055e62f
--- /dev/null
+++ b/workers/model-anigen/test_app_contract.py
@@ -0,0 +1,163 @@
+"""Smoke test for the served contract, run INSIDE the built image (it needs
+torch, FastAPI, PIL, google-cloud-storage, and the patched AniGen tree on
+PYTHONPATH):
+
+ docker build -t model-anigen workers/model-anigen
+ docker run --rm model-anigen python3 test_app_contract.py
+
+It needs no GPU, no weights, and no GCP credentials: the ASGI lifespan (which
+opens the GCS client and starts the model load) is deliberately not run, so
+every assertion below is about request handling, not inference.
+
+What it proves:
+
+ 1. The AniGen import chain resolves, and nvdiffrast is genuinely absent (see
+ patch_anigen.py): a module-level `import nvdiffrast` anywhere in the
+ resolved chain would mean the build-time patch regressed.
+ 2. The whole app module imports.
+ 3. The auth boundary rejects a wrong bearer on both authenticated routes.
+ 4. The unauthenticated probes (/ and /health) answer, and a dead pipeline
+ says so on both /health and /infer so the caller fails over.
+ 5. Request validation rejects an empty or oversized images array.
+ 6. The image decoder refuses everything the SSRF guard should refuse.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import os
+import sys
+
+os.environ.setdefault("API_KEY", "smoke-test-key")
+os.environ.setdefault("GCS_BUCKET", "smoke-test-bucket")
+
+KEY = os.environ["API_KEY"]
+
+PASSED = 0
+
+
+def check(name: str, condition: bool, detail: str = "") -> None:
+ global PASSED
+ if not condition:
+ print(f"FAIL {name} {detail}")
+ sys.exit(1)
+ PASSED += 1
+ print(f"ok {name}")
+
+
+def png_data_uri(color=(20, 20, 20, 255), size=(24, 24)) -> str:
+ from PIL import Image
+
+ img = Image.new("RGBA", size, color)
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+ return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
+
+
+_GPU_DRIVER_LIBS = ("libcuda.so", "libnvidia", "libnvrtc")
+
+
+def _is_gpu_driver_absence(exc: BaseException) -> bool:
+ text = str(exc)
+ if "cannot open shared object file" in text:
+ return any(lib in text for lib in _GPU_DRIVER_LIBS)
+ lowered = text.lower()
+ return "cuda" in lowered or "nvidia driver" in lowered or "no gpu" in lowered
+
+
+def main() -> int:
+ try:
+ import sys as _sys
+
+ for name in list(_sys.modules):
+ if name == "nvdiffrast" or name.startswith("nvdiffrast."):
+ check("nvdiffrast is not importable in this image", False, f"found {name} already imported")
+ break
+ else:
+ check("nvdiffrast module is absent before any import", "nvdiffrast" not in _sys.modules)
+
+ from anigen.pipelines import AnigenImageTo3DPipeline # noqa: F401
+ import anigen.utils.postprocessing_utils # noqa: F401
+ import anigen.renderers.mesh_renderer # noqa: F401
+
+ check("anigen import chain resolves", True)
+ check(
+ "nvdiffrast still not imported after loading anigen's modules",
+ "nvdiffrast" not in _sys.modules,
+ str(list(_sys.modules.keys()))[:200],
+ )
+ except ModuleNotFoundError as exc:
+ if "nvdiffrast" in str(exc):
+ check("anigen import chain resolves without nvdiffrast", False, str(exc))
+ else:
+ check("anigen import chain resolves", False, str(exc))
+ except Exception as exc: # noqa: BLE001 - classified below
+ if _is_gpu_driver_absence(exc):
+ print(f"skip anigen import chain (no GPU on this host): {type(exc).__name__}: {exc}")
+ else:
+ check("anigen import chain resolves", False, f"{type(exc).__name__}: {exc}")
+
+ import main as app_module
+ from fastapi.testclient import TestClient
+
+ client = TestClient(app_module.app)
+
+ bad = {"Authorization": "Bearer wrong-key"}
+ res = client.get("/tasks/does-not-exist", headers=bad)
+ check("GET /tasks rejects a wrong bearer", res.status_code == 401, f"got {res.status_code}")
+ res = client.post("/infer", headers=bad, json={"images": [png_data_uri()]})
+ check("POST /infer rejects a wrong bearer", res.status_code == 401, f"got {res.status_code}")
+
+ res = client.get("/health")
+ body = res.json()
+ check("GET /health answers 200", res.status_code == 200, f"got {res.status_code}")
+ check("health names the model", body.get("model") == "anigen", str(body)[:120])
+ res = client.get("/")
+ body = res.json()
+ check("GET / answers 200 for the keep-warm ping", res.status_code == 200, f"got {res.status_code}")
+ check("root names the service", body.get("service") == "model-anigen", str(body)[:120])
+ check("root lists the working endpoints", "POST /infer" in body.get("endpoints", []), str(body)[:160])
+
+ app_module._load_error = "internal error (ref deadbeefcafe)"
+ try:
+ res = client.get("/health")
+ body = res.json()
+ check("a dead pipeline makes /health answer 503", res.status_code == 503, f"got {res.status_code}")
+ check("a dead pipeline makes /health report ok:false", body.get("ok") is False, str(body)[:160])
+ check("health surfaces the load error", "deadbeefcafe" in str(body.get("load_error")), str(body)[:160])
+ res = client.post("/infer", headers={"Authorization": f"Bearer {KEY}"}, json={"images": [png_data_uri()]})
+ check("a dead pipeline makes /infer refuse with 503", res.status_code == 503, f"got {res.status_code}")
+ finally:
+ app_module._load_error = None
+ res = client.get("/health")
+ check("a live pipeline still answers /health 200", res.status_code == 200, f"got {res.status_code}")
+ check("a live pipeline reports ok:true", res.json().get("ok") is True, str(res.json())[:160])
+
+ good = {"Authorization": f"Bearer {KEY}"}
+ res = client.post("/infer", headers=good, json={"images": []})
+ check("POST /infer rejects an empty images array", res.status_code == 422, f"got {res.status_code}")
+ res = client.post("/infer", headers=good, json={"images": [png_data_uri()] * 7})
+ check("POST /infer rejects more than six images", res.status_code == 422, f"got {res.status_code}")
+
+ for label, src in [
+ ("cleartext http", "http://example.com/a.png"),
+ ("bare path", "/etc/passwd"),
+ ("loopback https", "https://127.0.0.1/a.png"),
+ ("cloud metadata", "https://169.254.169.254/latest/meta-data/"),
+ ("corrupt data uri", "data:image/png;base64,not-base64!!"),
+ ]:
+ try:
+ app_module._decode_image_bytes(src)
+ check(f"refuses {label}", False, "no exception raised")
+ except app_module.ImageSourceError:
+ check(f"refuses {label}", True)
+ except Exception as exc: # noqa: BLE001 - must be the caller-facing class
+ check(f"refuses {label}", False, f"raised {type(exc).__name__}: {exc}")
+
+ print(f"\n{PASSED} contract assertions passed")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/workers/model-anigen/test_patch_anigen.py b/workers/model-anigen/test_patch_anigen.py
new file mode 100644
index 000000000..0e8a754ab
--- /dev/null
+++ b/workers/model-anigen/test_patch_anigen.py
@@ -0,0 +1,170 @@
+"""Unit tests for patch_anigen.py against real excerpts of AniGen's own
+source (fetched from VAST-AI-Research/AniGen at the commit this worker
+pins, MIT licensed), not synthetic stand-ins: this proves the string
+surgery matches what the Docker build will actually see.
+
+No GPU, no torch, no AniGen checkout: `python3 test_patch_anigen.py`.
+"""
+from __future__ import annotations
+
+import sys
+import tempfile
+from pathlib import Path
+
+from patch_anigen import patch_mesh_renderer, patch_postprocessing_utils
+
+_FAILED = False
+
+
+def _check(cond: bool, label: str) -> None:
+ global _FAILED
+ status = "ok" if cond else "FAIL"
+ print(f" [{status}] {label}")
+ if not cond:
+ _FAILED = True
+
+
+def _check_raises(fn, exc_type, label: str) -> None:
+ try:
+ fn()
+ except exc_type:
+ _check(True, label)
+ return
+ except Exception as exc: # noqa: BLE001
+ _check(False, f"{label} (raised {type(exc).__name__}, not {exc_type.__name__})")
+ return
+ _check(False, f"{label} (did not raise)")
+
+
+# Real excerpt of anigen/utils/postprocessing_utils.py's top-of-file imports
+# (VAST-AI-Research/AniGen, MIT), trimmed to what the patch touches plus
+# enough surrounding context to prove it leaves everything else untouched.
+POSTPROCESSING_UTILS_REAL = '''from typing import *
+import gc
+import numpy as np
+import torch
+import utils3d
+import nvdiffrast.torch as dr
+from tqdm import tqdm
+import trimesh
+import trimesh.visual
+import xatlas
+import pyvista as pv
+from pymeshfix import _meshfix
+import igraph
+import cv2
+from PIL import Image
+from .random_utils import sphere_hammersley_sequence
+from .render_utils import render_multiview
+from ..renderers import GaussianRenderer
+from ..representations import Strivec, Gaussian, MeshExtractResult
+
+
+def _cuda_cleanup():
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+
+def _fill_holes(
+ verts,
+ faces,
+):
+ pass
+'''
+
+# Real excerpt of anigen/renderers/mesh_renderer.py (VAST-AI-Research/AniGen,
+# MIT): the module-level import plus the exact __init__ it must be relocated
+# into, and one dr.* call inside .render() that must survive untouched.
+MESH_RENDERER_REAL = '''import torch
+import nvdiffrast.torch as dr
+from easydict import EasyDict as edict
+from ..representations.mesh import MeshExtractResult
+import torch.nn.functional as F
+
+
+class MeshRenderer:
+ """
+ Renderer for the Mesh representation.
+ """
+ def __init__(self, rendering_options={}, device='cuda'):
+ self.rendering_options = edict({
+ "resolution": None,
+ })
+ self.rendering_options.update(rendering_options)
+ self.glctx = dr.RasterizeCudaContext(device=device)
+ self.device=device
+
+ def render(self, mesh, extrinsics, intrinsics):
+ rast, _ = dr.rasterize(self.glctx, None, None, (1, 1))
+ return rast
+'''
+
+
+def test_patch_postprocessing_utils_removes_only_the_top_level_import():
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "postprocessing_utils.py"
+ path.write_text(POSTPROCESSING_UTILS_REAL)
+ patch_postprocessing_utils(path)
+ patched = path.read_text()
+ _check("import nvdiffrast.torch as dr\n" not in patched, "top-level nvdiffrast import removed")
+ _check("from .render_utils import render_multiview" in patched, "unrelated import left intact")
+ _check("def _fill_holes(" in patched, "function bodies left intact")
+ compile(patched, str(path), "exec")
+ print(" [ok] patched file is syntactically valid Python")
+
+
+def test_patch_postprocessing_utils_raises_on_drifted_marker():
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "postprocessing_utils.py"
+ path.write_text(POSTPROCESSING_UTILS_REAL.replace(
+ "import nvdiffrast.torch as dr\n", "import nvdiffrast as nvd\n"
+ ))
+ _check_raises(
+ lambda: patch_postprocessing_utils(path), AssertionError,
+ "raises when the expected import line is not found (upstream drift)",
+ )
+
+
+def test_patch_mesh_renderer_relocates_import_into_init():
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "mesh_renderer.py"
+ path.write_text(MESH_RENDERER_REAL)
+ patch_mesh_renderer(path)
+ patched = path.read_text()
+ lines = patched.splitlines()
+ top_lines = "\n".join(lines[:6])
+ _check("import nvdiffrast" not in top_lines, "module-level nvdiffrast import removed from the top")
+ init_idx = next(i for i, l in enumerate(lines) if "def __init__" in l)
+ _check(
+ "import nvdiffrast.torch as dr" in lines[init_idx + 1],
+ "nvdiffrast import relocated to the first line of __init__",
+ )
+ _check("dr.rasterize(self.glctx, None, None, (1, 1))" in patched, "render() body left intact")
+ compile(patched, str(path), "exec")
+ print(" [ok] patched file is syntactically valid Python")
+
+
+def test_patch_mesh_renderer_raises_on_drifted_marker():
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "mesh_renderer.py"
+ path.write_text(MESH_RENDERER_REAL.replace(
+ "def __init__(self, rendering_options={}, device='cuda'):",
+ "def __init__(self, rendering_options={}):",
+ ))
+ _check_raises(
+ lambda: patch_mesh_renderer(path), AssertionError,
+ "raises when the expected __init__ signature is not found (upstream drift)",
+ )
+
+
+if __name__ == "__main__":
+ test_patch_postprocessing_utils_removes_only_the_top_level_import()
+ test_patch_postprocessing_utils_raises_on_drifted_marker()
+ test_patch_mesh_renderer_relocates_import_into_init()
+ test_patch_mesh_renderer_raises_on_drifted_marker()
+ if _FAILED:
+ print("\nFAILED")
+ sys.exit(1)
+ print("\nALL PASSED")
diff --git a/workers/model-anigen/worker_security.py b/workers/model-anigen/worker_security.py
new file mode 100644
index 000000000..802e1c32d
--- /dev/null
+++ b/workers/model-anigen/worker_security.py
@@ -0,0 +1,320 @@
+"""
+Shared security helpers for the three.ws GPU/CPU workers.
+
+This module is intentionally dependency-light (stdlib only) so it can be
+COPYed verbatim into every worker's Docker build context. The same canonical
+file lives in each worker directory; keep them byte-identical when editing.
+
+It provides three primitives that every worker needs:
+
+ 1. ``require_api_key`` - constant-time bearer-token check (timing-safe).
+ 2. ``fetch_remote_bytes`` - SSRF-hardened HTTP(S) fetch of caller-supplied
+ URLs: https-only, DNS resolution with private/loopback/link-local/metadata
+ IP rejection, per-hop redirect re-validation, and a bounded response size.
+ 3. ``safe_error`` - turns an exception into an opaque, correlation-id-tagged
+ message safe to persist/return to callers, after logging the full
+ traceback server-side only.
+"""
+
+from __future__ import annotations
+
+import hmac
+import ipaddress
+import logging
+import asyncio
+import socket
+import time
+import uuid
+from typing import Iterable, Optional
+from urllib.parse import urlsplit
+
+import httpx
+
+log = logging.getLogger("worker_security")
+
+# ── auth ────────────────────────────────────────────────────────────────────────
+
+
+def require_api_key(authorization: Optional[str], api_key: str) -> None:
+ """Validate a ``Bearer <token>`` header in constant time.
+
+ Raises ``PermissionError`` on any failure so callers can translate it into
+ the framework's 401 (FastAPI workers map this to HTTPException(401)).
+ """
+ # Fail closed on an empty configured key: with api_key == "" a bare
+ # "Bearer " header would otherwise strip to "" and compare equal.
+ if not api_key:
+ raise PermissionError("worker api key not configured")
+ if not authorization or not authorization.startswith("Bearer "):
+ raise PermissionError("missing bearer token")
+ token = authorization[len("Bearer "):].strip()
+ if not token:
+ raise PermissionError("missing bearer token")
+ # Compare bytes: hmac.compare_digest is timing-safe, and on str operands it
+ # raises TypeError for non-ASCII input, which would surface as a 500.
+ if not hmac.compare_digest(token.encode("utf-8"), api_key.encode("utf-8")):
+ raise PermissionError("invalid api key")
+
+
+# ── SSRF-hardened fetch ─────────────────────────────────────────────────────────
+
+# Default cap on a fetched response. Avatar selfies / driving audio / raw meshes
+# are comfortably under this; anything larger is almost certainly abuse.
+DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
+_MAX_REDIRECTS = 4
+
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
+# Hosts that enforce a User-Agent policy refuse httpx's default
+# "python-httpx/<version>": upload.wikimedia.org answers it with 403 Forbidden
+# and serves the same image to a descriptive agent (measured 2026-10-09, when
+# model-triposg failed every job whose reference photo was a Wikimedia link).
+# Callers may still pass their own User-Agent; it replaces this one.
+DEFAULT_USER_AGENT = "three.ws-worker/1.0 (+https://three.ws)"
+
+
+def _fetch_headers(headers: Optional[dict]) -> dict:
+ """Caller headers over the default User-Agent, matched case-insensitively."""
+ merged = dict(headers or {})
+ if not any(str(k).lower() == "user-agent" for k in merged):
+ merged["User-Agent"] = DEFAULT_USER_AGENT
+ return merged
+
+
+class UnsafeUrlError(ValueError):
+ """Raised when a URL targets a disallowed scheme or a private/internal IP."""
+
+
+def _is_blocked_ip(ip: ipaddress._BaseAddress) -> bool:
+ """Reject any address that could reach internal infrastructure.
+
+ Covers loopback, RFC1918 private ranges, link-local (incl. the cloud
+ metadata server at 169.254.169.254), unique-local IPv6 (fc00::/7),
+ multicast, reserved, unspecified, and any other non-global address (CGNAT
+ 100.64.0.0/10 included). IPv4-mapped IPv6 addresses
+ are unwrapped first so a mapped private v4 can't slip through.
+ """
+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+ ip = ip.ipv4_mapped
+ return (
+ ip.is_private
+ or ip.is_loopback
+ or ip.is_link_local
+ or ip.is_multicast
+ or ip.is_reserved
+ or ip.is_unspecified
+ # Anything not globally routable, which also covers the CGNAT shared
+ # space 100.64.0.0/10 that is_private does not flag.
+ or not ip.is_global
+ )
+
+
+def _resolve_safe_host(host: str) -> None:
+ """DNS-resolve ``host`` and raise UnsafeUrlError if ANY address is internal.
+
+ Resolving and checking every returned address closes the gap where a name
+ resolves to both a public and a private address.
+ """
+ if not host:
+ raise UnsafeUrlError("missing host")
+ try:
+ infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
+ except socket.gaierror as exc:
+ raise UnsafeUrlError(f"could not resolve host: {host}") from exc
+
+ addrs = {info[4][0] for info in infos}
+ if not addrs:
+ raise UnsafeUrlError(f"no addresses for host: {host}")
+ for addr in addrs:
+ ip = ipaddress.ip_address(addr)
+ if _is_blocked_ip(ip):
+ raise UnsafeUrlError(f"host resolves to a disallowed address: {host}")
+
+
+def assert_safe_url(url: str, *, allow_http: bool = False) -> str:
+ """Validate scheme + resolved IPs for a single URL. Returns it unchanged.
+
+ https is required by default; ``allow_http`` exists only for callers that
+ legitimately need cleartext (not used by the workers).
+ """
+ parts = urlsplit(url)
+ scheme = parts.scheme.lower()
+ allowed = ("https", "http") if allow_http else ("https",)
+ if scheme not in allowed:
+ raise UnsafeUrlError(f"scheme not allowed: {scheme or '(none)'}")
+ if not parts.hostname:
+ raise UnsafeUrlError("missing host")
+ _resolve_safe_host(parts.hostname)
+ return url
+
+
+def fetch_remote_bytes(
+ url: str,
+ *,
+ timeout: float = 30.0,
+ max_bytes: int = DEFAULT_MAX_BYTES,
+ allow_http: bool = False,
+ headers: Optional[dict] = None,
+) -> bytes:
+ """Fetch ``url`` safely and return its body bytes.
+
+ Hardening applied:
+ * https-only (unless ``allow_http``).
+ * every URL - including each redirect target - is DNS-resolved and
+ rejected if it maps to a private/loopback/link-local/metadata address.
+ * redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
+ re-validated; httpx's own redirect following is disabled.
+ * the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
+
+ Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
+ ``ValueError`` for transport or size failures.
+ """
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
+ current = assert_safe_url(url, allow_http=allow_http)
+ with httpx.Client(follow_redirects=False, timeout=timeout) as client:
+ for _ in range(_MAX_REDIRECTS + 1):
+ with client.stream("GET", current, headers=_fetch_headers(headers)) as resp:
+ if resp.is_redirect:
+ location = resp.headers.get("location")
+ if not location:
+ raise UnsafeUrlError("redirect without Location header")
+ current = assert_safe_url(
+ str(resp.url.join(location)), allow_http=allow_http
+ )
+ continue
+ resp.raise_for_status()
+ declared = resp.headers.get("content-length")
+ if declared is not None and declared.isdigit() and int(declared) > max_bytes:
+ raise ValueError(f"response too large: {declared} bytes")
+ chunks: list[bytes] = []
+ total = 0
+ for chunk in resp.iter_bytes(65536):
+ total += len(chunk)
+ if total > max_bytes:
+ raise ValueError(f"response exceeded {max_bytes} bytes")
+ chunks.append(chunk)
+ return b"".join(chunks)
+ raise UnsafeUrlError(f"too many redirects (>{_MAX_REDIRECTS})")
+
+
+async def fetch_remote_bytes_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int = DEFAULT_MAX_BYTES,
+ allow_http: bool = False,
+ headers: Optional[dict] = None,
+) -> bytes:
+ """Async variant of ``fetch_remote_bytes`` using a caller-provided client.
+
+ The provided ``client`` MUST be constructed with ``follow_redirects=False``
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
+ """
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
+ current = assert_safe_url(url, allow_http=allow_http)
+ for _ in range(_MAX_REDIRECTS + 1):
+ async with client.stream("GET", current, headers=_fetch_headers(headers)) as resp:
+ if resp.is_redirect:
+ location = resp.headers.get("location")
+ if not location:
+ raise UnsafeUrlError("redirect without Location header")
+ current = assert_safe_url(
+ str(resp.url.join(location)), allow_http=allow_http
+ )
+ continue
+ resp.raise_for_status()
+ declared = resp.headers.get("content-length")
+ if declared is not None and declared.isdigit() and int(declared) > max_bytes:
+ raise ValueError(f"response too large: {declared} bytes")
+ chunks: list[bytes] = []
+ total = 0
+ async for chunk in resp.aiter_bytes(65536):
+ total += len(chunk)
+ if total > max_bytes:
+ raise ValueError(f"response exceeded {max_bytes} bytes")
+ chunks.append(chunk)
+ return b"".join(chunks)
+ raise UnsafeUrlError(f"too many redirects (>{_MAX_REDIRECTS})")
+
+
+# ── opaque error handling ───────────────────────────────────────────────────────
+
+
+def safe_error(exc: BaseException, *, context: str = "") -> str:
+ """Log the full traceback server-side and return an opaque client message.
+
+ Never leak ``traceback.format_exc()`` (absolute paths, library versions,
+ internal hostnames, fragments of upstream responses) to callers. A short
+ correlation id is embedded in both the log line and the returned string so
+ operators can join a client-reported failure to the server log.
+ """
+ correlation_id = uuid.uuid4().hex[:12]
+ label = f"{context} " if context else ""
+ log.exception("%serror [correlation_id=%s]", label, correlation_id)
+ return f"internal error (ref {correlation_id})"