Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(best3d): add PartCrafter and HoloPart GPU workers for editable parts
nirholasd3edf5a25daf
diff --git a/scripts/check-vendored-workers.mjs b/scripts/check-vendored-workers.mjsindex 5e9abd09d..07e236ab8 100644--- a/scripts/check-vendored-workers.mjs+++ b/scripts/check-vendored-workers.mjs@@ -79,13 +79,50 @@ for (const file of VENDORED) {}}-if (failures.length) {- console.error('[check-vendored-workers] vendored worker files have drifted:\n');- for (const failure of failures) console.error(` ${failure}\n`);- console.error('Fix the canonical copy, mirror it to every worker, and rerun.');+// gltf_meshopt.py shells out to a `gltfpack` binary on PATH (see its+// GLTFPACK_BIN constant). A worker that vendors the module but never stages+// the binary in its Dockerfile passes the build-time unit test (which mocks+// nothing, but also never feeds it meshopt-compressed input) and then fails+// every real meshopt-compressed request at runtime with "the gltfpack+// decoder is not installed". Catch that gap here instead of in production.+const missingGltfpack = [];+for (const dir of dirs) {+ const modulePath = join(WORKERS_DIR, dir, 'gltf_meshopt.py');+ try {+ statSync(modulePath);+ } catch {+ continue;+ }+ const dockerfilePath = join(WORKERS_DIR, dir, 'Dockerfile');+ let dockerfile;+ try {+ dockerfile = readFileSync(dockerfilePath, 'utf8');+ } catch {+ missingGltfpack.push(`${dir}: vendors gltf_meshopt.py but has no Dockerfile at ${dockerfilePath}`);+ continue;+ }+ if (!dockerfile.includes('COPY --from=gltfpack')) {+ missingGltfpack.push(+ `${dir}: vendors gltf_meshopt.py but ${dockerfilePath} never stages the gltfpack binary ` ++ `(no "COPY --from=gltfpack" line; see workers/segment/Dockerfile for the pattern)`,+ );+ }+}++if (failures.length || missingGltfpack.length) {+ if (failures.length) {+ console.error('[check-vendored-workers] vendored worker files have drifted:\n');+ for (const failure of failures) console.error(` ${failure}\n`);+ console.error('Fix the canonical copy, mirror it to every worker, and rerun.');+ }+ if (missingGltfpack.length) {+ console.error('[check-vendored-workers] gltf_meshopt.py is vendored without its gltfpack binary:\n');+ for (const failure of missingGltfpack) console.error(` ${failure}\n`);+ }process.exit(1);}console.log(- `[check-vendored-workers] OK: ${checked} vendored file copies across ${dirs.length} workers are byte-identical`,+ `[check-vendored-workers] OK: ${checked} vendored file copies across ${dirs.length} workers are byte-identical, ` ++ 'and every worker that vendors gltf_meshopt.py stages the gltfpack binary',);diff --git a/workers/model-holopart/Dockerfile b/workers/model-holopart/Dockerfilenew file mode 100644index 000000000..7433cfbac--- /dev/null+++ b/workers/model-holopart/Dockerfile@@ -0,0 +1,85 @@+# HoloPart (VAST-AI-Research, MIT, LICENSE read 2026-10-11): generative 3D+# part amodal segmentation. Given an already part-segmented mesh (one named+# geometry per part, e.g. workers/segment's output), completes each part's+# occluded geometry while preserving its name. Its own requirements.txt pins+# diffusers/transformers/diso/torch-cluster; torch is pinned separately here+# to match the CUDA image base, same pattern as workers/model-anigen and+# workers/model-partcrafter.+#+# gltfpack (zeux/meshoptimizer, MIT) is pinned by release tag and checksum,+# the same pin workers/segment ships: gltf_meshopt.py shells out to it to+# decode EXT_meshopt_compression before trimesh reads a caller's mesh.+FROM python:3.11-slim AS gltfpack+ARG GLTFPACK_VERSION=v1.2+ARG GLTFPACK_SHA256=ebc236f5f6c08c7e5c5750476a187d24805d44d8c680449c4b7369c333f817b1+RUN apt-get update && apt-get install -y --no-install-recommends curl unzip ca-certificates \+ && curl -fsSL -o /tmp/gltfpack.zip \+ https://github.com/zeux/meshoptimizer/releases/download/${GLTFPACK_VERSION}/gltfpack-ubuntu.zip \+ && echo "${GLTFPACK_SHA256} /tmp/gltfpack.zip" | sha256sum -c - \+ && unzip -j /tmp/gltfpack.zip gltfpack -d /usr/local/bin \+ && chmod +x /usr/local/bin/gltfpack \+ && /usr/local/bin/gltfpack -v++FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04++ENV PYTHONDONTWRITEBYTECODE=1 \+ PYTHONUNBUFFERED=1 \+ PIP_NO_CACHE_DIR=1 \+ DEBIAN_FRONTEND=noninteractive \+ PYTHONPATH=/app/holopart-src \+ HF_HUB_OFFLINE=1 \+ TRANSFORMERS_OFFLINE=1 \+ HF_HUB_DISABLE_TELEMETRY=1 \+ DO_NOT_TRACK=1++RUN apt-get update && apt-get install -y --no-install-recommends \+ python3.10 python3.10-dev python3-pip \+ git curl build-essential ninja-build \+ libgl1 libglib2.0-0 libgomp1 \+ && rm -rf /var/lib/apt/lists/* \+ && ln -sf /usr/bin/python3.10 /usr/bin/python3 \+ && ln -sf /usr/bin/python3.10 /usr/bin/python \+ && pip install --upgrade pip setuptools wheel++COPY --from=gltfpack /usr/local/bin/gltfpack /usr/local/bin/gltfpack++# 1. torch, its own cached layer (every extension below links against it;+# torch-cluster below needs a matching prebuilt wheel index).+RUN pip install torch==2.4.1 torchvision==0.19.1 --index-url https://download.pytorch.org/whl/cu121++# 2. HoloPart, pinned to the commit read on 2026-10-11+# (https://github.com/VAST-AI-Research/HoloPart/commit/48dd632c9a7412e6688573bb4f153e0ade7e7325).+ARG HOLOPART_COMMIT=48dd632c9a7412e6688573bb4f153e0ade7e7325+RUN git clone https://github.com/VAST-AI-Research/HoloPart.git /app/holopart-src \+ && git -C /app/holopart-src checkout ${HOLOPART_COMMIT} \+ && rm -rf /app/holopart-src/.git++# 3. HoloPart's own requirements (minus numpy's hard == pin, which conflicts+# with this worker's trimesh/pymeshlab stack; torch-cluster needs the+# cu121-matched wheel index).+RUN grep -viE '^numpy([=<> ]|$)' /app/holopart-src/requirements.txt > /tmp/holopart-reqs.txt \+ && grep -viE '^torch-cluster([=<> ]|$)' /tmp/holopart-reqs.txt > /tmp/holopart-reqs2.txt \+ && pip install -r /tmp/holopart-reqs2.txt "numpy>=1.24,<2" \+ && pip install torch-cluster -f https://data.pyg.org/whl/torch-2.4.1+cu121.html++WORKDIR /app+COPY requirements.txt /app/requirements.txt+RUN pip install -r /app/requirements.txt++COPY worker_security.py storage_backend.py gltf_meshopt.py pipeline.py main.py entrypoint.sh /app/+COPY test_gltf_meshopt.py test_app_contract.py /app/+RUN chmod +x /app/entrypoint.sh++# Build-time guard: served-contract tests need no GPU and no checkpoints, so+# they run here and the image does not ship if the import chain or request+# handling broke.+RUN python3 /app/test_gltf_meshopt.py && python3 /app/test_app_contract.py++# Weights are staged in GCS (stage-assets.sh) and mounted at /weights by+# Cloud Run; entrypoint.sh copies them into HOLOPART_WEIGHTS_ROOT, the+# literal path pipeline.py's from_pretrained call uses, so inference needs no+# Hugging Face egress at runtime.++EXPOSE 8080++CMD ["/app/entrypoint.sh"]diff --git a/workers/model-holopart/README.md b/workers/model-holopart/README.mdnew file mode 100644index 000000000..38a64847e--- /dev/null+++ b/workers/model-holopart/README.md@@ -0,0 +1,122 @@+# model-holopart: complete-hidden-parts worker++`workers/segment` splits a mesh at its visible seams, but each resulting part+is an open shell where it touched a neighbor or was hidden from every camera+angle. This worker, built on+[HoloPart](https://github.com/VAST-AI-Research/HoloPart) (VAST-AI-Research,+MIT, LICENSE read 2026-10-11), takes an already part-segmented mesh and+generates the missing geometry for every part, closing each one into a+complete, standalone shape. This is the "Complete hidden parts" lane behind+segment (work order 070); `model-partcrafter` is the companion worker that+generates parts straight from a 2D image instead of completing an+already-segmented mesh.++## How it works++`HoloPartPipeline` (imported directly, in-process: HoloPart is a real Python+package on its own checkout, not a subprocess CLI) reads a `trimesh.Scene`+whose geometries are already the named parts (exactly the shape+`workers/segment` emits, one node per part), builds per-part and whole-object+point-cloud conditioning tensors, and decodes a geometrically complete mesh+for every part. See `pipeline.py` for the exact call, adapted directly from+upstream's own `scripts/inference_holopart.py`.++Upstream's script already writes `part_mesh.name = part_id_list[i]` on every+output part, so node names survive this pass with zero extra engineering:+the node name a caller passed in is the exact node name that comes back.+That is the stability guarantee work order 070's Task 3 (and work order 078,+"keep the rig") depends on. This worker re-tints each completed part with the+same golden-ratio hue stepping `workers/segment` and `model-partcrafter` use,+so a completed part keeps the color a caller already associated with its id.++## API++```+POST /infer { mesh_url, seed?, job_id? }+ -> 202 { task_id, status: "queued" } (Bearer API_KEY)+GET /tasks/:id -> { task_id, status, result_gcs_url?, result_url?,+ parts?, part_count?, error? }+GET /results/:id.glb -> completed GLB (local OUTPUT_DIR mode only)+GET /health -> { ok, model, gpu_available, gpu_name, load_error }+```++`mesh_url` must point at a GLB/GLTF with at least two named part geometries+(the output of `workers/segment`, `model-partcrafter`, or any other+SAMPart3D/SAMesh-style splitter); FBX and single-geometry meshes are refused+up front with a caller-facing message rather than failing opaquely on the+GPU. A meshopt-compressed input (`EXT_meshopt_compression`, what most+three.ws avatars ship) is transcoded before load, same as every other+caller-glTF-reading worker.++## Deploy++One-time weight staging (under `gs://three-ws-model-weights/holopart/`):++```bash+bash workers/model-holopart/stage-assets.sh+```++Build + deploy (Cloud Build pins the `three-ws-build@` service account):++```bash+gcloud builds submit --config workers/model-holopart/cloudbuild.yaml \+ --substitutions=SHORT_SHA=manual$(date +%s) .+```++Service: `model-holopart`, us-central1, 1x L4 (`--no-gpu-zonal-redundancy`),+scale-to-zero by default against the shared per-region L4 quota in+`docs/ops/gcp-credits-plan.md`.++Then point the Forge API at it:++```bash+gcloud run services update three-ws-api --region us-central1 \+ --update-env-vars MODEL_HOLOPART_URL=$(gcloud run services describe \+ model-holopart --region us-central1 --format='value(status.url)')+```++`MODEL_HOLOPART_URL` is unset by default, so until that command runs, the+`holopart` Forge backend stays inactive (same invariant as every other+self-hosted lane: it is opt-in only, never a default or a failover target).++## Local tests (no GPU needed)++```bash+cd workers/model-holopart+python3 -m py_compile main.py pipeline.py storage_backend.py worker_security.py gltf_meshopt.py test_gltf_meshopt.py test_app_contract.py+python3 test_gltf_meshopt.py+```++`test_app_contract.py` additionally proves the served contract (auth+boundary, request validation, SSRF guard, health reporting, the+fewer-than-two-parts guard), but it needs torch, FastAPI, trimesh, and the+vendored HoloPart checkout on `PYTHONPATH`, so it runs inside the built image+(`docker build -t model-holopart workers/model-holopart`, which runs it as a+build-time gate; see the Dockerfile tail) rather than bare on a dev machine.++## Licensing++HoloPart is MIT (LICENSE read 2026-10-11).++## Environment++| Var | Default | Meaning |+|---|---|---|+| `API_KEY` | (secret) | shared bearer secret (`avatar-reconstruction-key`) |+| `GCS_BUCKET` | (optional) | output bucket for completed GLBs; local `OUTPUT_DIR` storage when unset |+| `OUTPUT_DIR` | `/output` | local task/result volume when `GCS_BUCKET` is unset |+| `HOLOPART_WEIGHTS_ROOT` | `/app/pretrained_weights/HoloPart` | local root `entrypoint.sh` stages weights into |+| `MAX_CONCURRENT` | `1` | parallel inference jobs per instance |+| `WEIGHTS_ROOT` | `/weights/holopart` | staged weights, read by `entrypoint.sh` |+| `MODEL_LOAD_ATTEMPTS` / `MODEL_LOAD_RETRY_BASE_S` / `MODEL_LOAD_RETRY_CAP_S` | `4` / `15` / `120` | model-load retry budget |+| `MESH_FETCH_TIMEOUT_S` | `60` | per-attempt mesh fetch timeout |++## Failure modes++| Reported `error` | Cause | Fix |+|---|---|---|+| `refused to fetch mesh ...` / `mesh source ... rejected` | bad, unreachable, or unsafe `mesh_url` | fix the URL; rejected before touching the GPU |+| `HoloPart needs an already part-segmented GLB or GLTF: ...` | FBX input, or a mesh with fewer than two named geometries | segment the mesh first (`workers/segment` or `model-partcrafter`) |+| `HoloPart completed no usable parts from this mesh` | every decoded part came back empty | retry; if persistent, check the input parts are each a coherent, non-degenerate shape |+| `pipeline unavailable: ...` / `pipeline not ready` | weight load failed or is still in progress | check `/health`'s `load_error`; retry once the instance reports ready |+| `internal error (ref <id>)` | anything unexpected | grep the service logs for the correlation id |diff --git a/workers/model-holopart/cloudbuild.yaml b/workers/model-holopart/cloudbuild.yamlnew file mode 100644index 000000000..f1b38c3e4--- /dev/null+++ b/workers/model-holopart/cloudbuild.yaml@@ -0,0 +1,80 @@+# Build + deploy the HoloPart "complete hidden parts" worker.+# Manual submit (SHORT_SHA is only auto-set by triggers):+# gcloud builds submit --config workers/model-holopart/cloudbuild.yaml \+# --substitutions=SHORT_SHA=manual$(date +%s) .+serviceAccount: projects/aerial-vehicle-466722-p5/serviceAccounts/[email protected]++substitutions:+ _REGION: us-central1+ _RUN_SA: avatar-reconstruction-sa@aerial-vehicle-466722-p5.iam.gserviceaccount.com+ _SERVICE: model-holopart+ _REPO: model-holopart+ _GCS_BUCKET: three-ws-avatar-reconstructions+ _WEIGHTS_BUCKET: three-ws-model-weights+ # Scale-to-zero by default, same reasoning as workers/model-anigen and+ # workers/model-partcrafter: the us-central1 L4 grant is shared, and a warm+ # floor here is a quota decision, not a default. Raise with `gcloud run+ # services update model-holopart --min-instances=1` once the grant allows.+ _MIN_INSTANCES: "0"+ _MAX_INSTANCES: "2"++steps:+ - id: build+ name: gcr.io/cloud-builders/docker+ dir: workers/model-holopart+ args:+ - build+ - --tag+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA+ - --tag+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest+ - --cache-from+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest+ - .++ - id: push+ name: gcr.io/cloud-builders/docker+ args:+ - push+ - --all-tags+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server++ - id: deploy+ name: gcr.io/google.com/cloudsdktool/cloud-sdk:slim+ entrypoint: gcloud+ args:+ - run+ - deploy+ - ${_SERVICE}+ - --image=${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA+ - --region=${_REGION}+ - --service-account=${_RUN_SA}+ - --platform=managed+ - --port=8080+ - --gpu=1+ - --gpu-type=nvidia-l4+ - --no-gpu-zonal-redundancy+ - --no-cpu-throttling+ - --cpu=4+ - --memory=16Gi+ - --min-instances=${_MIN_INSTANCES}+ - --max-instances=${_MAX_INSTANCES}+ - --timeout=900+ - --startup-probe=tcpSocket.port=8080,initialDelaySeconds=0,periodSeconds=60,timeoutSeconds=60,failureThreshold=15+ - --cpu-boost+ - --allow-unauthenticated+ - --add-volume=name=weights,type=cloud-storage,bucket=${_WEIGHTS_BUCKET}+ - --add-volume-mount=volume=weights,mount-path=/weights+ - --set-env-vars=GCS_BUCKET=${_GCS_BUCKET},MAX_CONCURRENT=1,WEIGHTS_ROOT=/weights/holopart+ - --set-secrets=API_KEY=avatar-reconstruction-key:latest+ - --liveness-probe=httpGet.path=/health,initialDelaySeconds=60,periodSeconds=60,timeoutSeconds=10,failureThreshold=3++images:+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest++options:+ machineType: E2_HIGHCPU_32+ logging: CLOUD_LOGGING_ONLY++timeout: 3600sdiff --git a/workers/model-holopart/entrypoint.sh b/workers/model-holopart/entrypoint.shnew file mode 100644index 000000000..520c82dd1--- /dev/null+++ b/workers/model-holopart/entrypoint.sh@@ -0,0 +1,24 @@+#!/bin/sh+# Wire staged HoloPart weights into the local path pipeline.py loads from+# (HOLOPART_WEIGHTS_ROOT), then serve.+#+# Checkpoints are COPIED from the FUSE-mounted bucket to local disk rather+# than symlinked, same reasoning as workers/model-anigen and+# workers/model-partcrafter: torch.load over gcsfuse does slow random reads,+# and a sequential cp of a few GB is fast and deterministic.+set -eu++WEIGHTS_ROOT="${WEIGHTS_ROOT:-/weights/holopart}"+HOLOPART_WEIGHTS_ROOT="${HOLOPART_WEIGHTS_ROOT:-/app/pretrained_weights/HoloPart}"++if [ ! -e "$WEIGHTS_ROOT/HoloPart" ]; then+ echo "fatal: expected staged weights at $WEIGHTS_ROOT/HoloPart (run workers/model-holopart/stage-assets.sh)" >&2+ exit 1+fi++mkdir -p "$(dirname "$HOLOPART_WEIGHTS_ROOT")"+rm -rf "$HOLOPART_WEIGHTS_ROOT"+echo "staging $WEIGHTS_ROOT/HoloPart -> $HOLOPART_WEIGHTS_ROOT"+cp -r "$WEIGHTS_ROOT/HoloPart" "$HOLOPART_WEIGHTS_ROOT"++exec uvicorn main:app --host 0.0.0.0 --port 8080 --workers 1 --timeout-keep-alive 300diff --git a/workers/model-holopart/gltf_meshopt.py b/workers/model-holopart/gltf_meshopt.pynew file mode 100644index 000000000..10f7ba4ad--- /dev/null+++ b/workers/model-holopart/gltf_meshopt.py@@ -0,0 +1,107 @@+"""EXT_meshopt_compression decoding for caller-supplied glTF input.++Vendored byte-identical into every worker that loads a mesh a caller handed it,+exactly like worker_security.py: each worker's Docker build context is its own+directory, so `../` is unreachable. `npm run check:vendored` fails if the copies+drift apart.++Why it exists: trimesh has no decoder for EXT_meshopt_compression. A compressed+asset stores its geometry in compressed buffer views and declares a second,+empty "fallback" buffer that only exists once something decodes into it, so+trimesh dies on the bufferView lookup with a bare `IndexError: list index out of+range`. Meshopt is what gltfpack emits and what most three.ws avatars ship as,+so this was a hard failure on our own assets (verified 2026-08-13:+/avatars/michelle.glb failed every stylize filter in production). gltfpack is+the same project's CLI and decodes what it encoded, so the fix is to transcode+the asset to plain glTF before any reader touches it.++The binary is pinned by release tag and checksum in each worker's Dockerfile.+Set GLTFPACK_BIN to run the path against your own copy locally.+"""++from __future__ import annotations++import json+import logging+import os+import subprocess+import tempfile+from pathlib import Path+from typing import Optional++log = logging.getLogger("gltf-meshopt")++MESHOPT_EXTENSION = "EXT_meshopt_compression"+GLTFPACK_BIN = os.environ.get("GLTFPACK_BIN", "gltfpack")+GLTFPACK_TIMEOUT_S = int(os.environ.get("GLTFPACK_TIMEOUT_S", "120"))+++def gltf_document(data: bytes, suffix: str = ".glb") -> Optional[dict]:+ """The glTF JSON of a .glb/.gltf payload, or None when it is neither."""+ if suffix == ".gltf":+ try:+ return json.loads(data)+ except ValueError:+ return None+ if suffix != ".glb" or len(data) < 20 or data[:4] != b"glTF":+ return None+ chunk_length = int.from_bytes(data[12:16], "little")+ if data[16:20] != b"JSON":+ return None+ try:+ return json.loads(data[20:20 + chunk_length])+ except ValueError:+ return None+++def uses_meshopt(document: Optional[dict]) -> bool:+ """True when the asset needs a meshopt decode before any reader can use it.++ Checked against both extension lists: an asset that only *uses* the+ extension still stores its geometry in the compressed views, and its+ fallback buffer stays empty until something decodes into it.+ """+ if not document:+ return False+ declared = set(document.get("extensionsRequired") or []) | set(document.get("extensionsUsed") or [])+ return MESHOPT_EXTENSION in declared+++def transcode_meshopt(data: bytes, suffix: str = ".glb") -> bytes:+ """Rewrite a meshopt-compressed asset as a plain GLB with gltfpack.++ `-noq` keeps the source quantization instead of re-quantizing on the way+ out, so downstream geometry is derived from what the author shipped.+ """+ with tempfile.TemporaryDirectory() as tmpdir:+ src = Path(tmpdir) / f"input{suffix}"+ dst = Path(tmpdir) / "decoded.glb"+ src.write_bytes(data)+ try:+ proc = subprocess.run(+ [GLTFPACK_BIN, "-i", str(src), "-o", str(dst), "-noq"],+ capture_output=True,+ timeout=GLTFPACK_TIMEOUT_S,+ )+ except FileNotFoundError as exc:+ raise ValueError(+ f"input uses {MESHOPT_EXTENSION} and the gltfpack decoder is not installed"+ ) from exc+ except subprocess.TimeoutExpired as exc:+ raise ValueError(f"{MESHOPT_EXTENSION} decode timed out") from exc+ if proc.returncode != 0 or not dst.exists():+ detail = (proc.stderr or b"").decode("utf-8", "replace").strip().splitlines()+ raise ValueError(f"{MESHOPT_EXTENSION} decode failed: {detail[-1][:200] if detail else 'no output'}")+ return dst.read_bytes()+++def decode_if_meshopt(data: bytes, suffix: str = ".glb") -> tuple[bytes, str]:+ """Return (data, suffix) ready for trimesh, decoding meshopt when present.++ A payload that is not compressed glTF is handed back untouched, so this is+ safe to call on every input path.+ """+ if not uses_meshopt(gltf_document(data, suffix)):+ return data, suffix+ log.info("input declares %s; decoding with gltfpack before load", MESHOPT_EXTENSION)+ return transcode_meshopt(data, suffix), ".glb"diff --git a/workers/model-holopart/main.py b/workers/model-holopart/main.pynew file mode 100644index 000000000..8179061be--- /dev/null+++ b/workers/model-holopart/main.py@@ -0,0 +1,457 @@+"""+HoloPart inference service: complete the occluded geometry of an+already part-segmented mesh (VAST-AI-Research/HoloPart, MIT; generative 3D+part amodal segmentation).++This is the Forge "Complete hidden parts" lane behind the segment lane (work+order 070): where workers/segment only cuts a mesh at its visible seams,+leaving each part's hidden/occluded side an open shell, this worker fills+each named part back in to a geometrically complete, closed mesh, with no+extra reconstruction call needed. Node names are preserved verbatim from the+input (`part_mesh.name = part_id_list[i]` in upstream's own script), which is+exactly what work order 070's Task 3 (stable node names) requires.++API contract (consumed by the Forge router, same task shape as the other+mesh-in GPU workers, e.g. workers/segment):+ POST /infer { mesh_url: https URL to an ALREADY part-segmented GLB+ (one named geometry per part, as workers/segment emits),+ seed?: int, job_id?: str }+ -> 202 { task_id, status: "queued" }++ GET /tasks/:id -> { task_id, status, result_gcs_url?, result_url?,+ parts?, part_count?, error? }++ GET /results/:id.glb -> completed GLB when OUTPUT_DIR local storage is active++ GET /health -> { ok, model, gpu_available, load_error, load_attempts }.+ 503 with ok:false once the model load has spent its+ retry budget, so a dead instance is drained.++ GET / -> { service, model, ready, endpoints }. Unauthenticated+ descriptor for the platform's warmth probe.++Weights: gs://three-ws-model-weights/holopart (workers/model-holopart/stage-assets.sh).++Environment variables (README.md carries the full table):+ API_KEY shared bearer secret+ GCS_BUCKET optional Cloud Storage bucket for output meshes+ OUTPUT_DIR local task and result volume when GCS_BUCKET is unset+ HOLOPART_WEIGHTS_ROOT local checkpoint root entrypoint.sh stages into (default: /app/pretrained_weights/HoloPart)+ MAX_CONCURRENT max parallel inferences (default: 1)+ MESH_FETCH_TIMEOUT_S per-attempt mesh fetch timeout (default: 60)+ MODEL_LOAD_ATTEMPTS model-load attempts before the error latches (default: 4)+ MODEL_LOAD_RETRY_BASE_S / MODEL_LOAD_RETRY_CAP_S load-retry backoff bounds+"""++from __future__ import annotations++import asyncio+import gc+import json+import logging+import os+import time+import uuid+from contextlib import asynccontextmanager+from pathlib import Path+from typing import Optional+from urllib.parse import urlsplit++import httpx+import torch+from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Response+from fastapi.responses import FileResponse+from google.api_core.exceptions import NotFound+from google.cloud import storage+from pydantic import BaseModel, Field++from gltf_meshopt import decode_if_meshopt+from pipeline import PartCompletionError, run_holopart+from storage_backend import LocalStorage+from worker_security import (+ UnsafeUrlError,+ fetch_remote_bytes,+ require_api_key,+ safe_error,+)++logging.basicConfig(+ level=logging.INFO,+ format="%(asctime)s %(levelname)s %(name)s %(message)s",+)+log = logging.getLogger("model-holopart")++API_KEY = os.environ["API_KEY"]+GCS_BUCKET = os.environ.get("GCS_BUCKET", "").strip()+OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "/output")+HOLOPART_WEIGHTS_ROOT = os.environ.get("HOLOPART_WEIGHTS_ROOT", "/app/pretrained_weights/HoloPart")+MAX_CONCURRENT = int(os.environ.get("MAX_CONCURRENT", "1"))+MODEL_LOAD_ATTEMPTS = int(os.environ.get("MODEL_LOAD_ATTEMPTS", "4"))+MODEL_LOAD_RETRY_BASE_S = float(os.environ.get("MODEL_LOAD_RETRY_BASE_S", "15"))+MODEL_LOAD_RETRY_CAP_S = float(os.environ.get("MODEL_LOAD_RETRY_CAP_S", "120"))+MESH_FETCH_TIMEOUT_S = float(os.environ.get("MESH_FETCH_TIMEOUT_S", "60"))+FETCH_ATTEMPTS = 3+MAX_MESH_BYTES = 128 * 1024 * 1024++# FBX has no part-name convention HoloPart can read and trimesh has no FBX+# reader, same reasoning as workers/segment.+SUPPORTED_INPUT_FORMATS = {".glb", ".gltf"}+SUPPORTED_INPUT_LABEL = "GLB or GLTF"+UNSUPPORTED_FORMAT_MESSAGE = (+ f"HoloPart needs an already part-segmented {SUPPORTED_INPUT_LABEL} (e.g. the output of "+ "the segment worker): one named geometry per part. Segment the mesh first."+)+FBX_BINARY_MAGIC = b"Kaydara FBX Binary \x00"+FBX_ASCII_HEADER = b"; FBX"++_bucket: Optional[storage.Bucket] = None+_local_storage: Optional[LocalStorage] = None+_sem: Optional[asyncio.Semaphore] = None+_ready: Optional[asyncio.Event] = None+_load_error: Optional[str] = None+_load_attempts = 0+_pipeline_loaded = False+_tasks: dict[str, dict] = {}+_task_locks: dict[str, asyncio.Lock] = {}+++def _load_pipeline_sync() -> None:+ global _pipeline_loaded+ from pipeline import load_pipeline++ log.info("Loading HoloPart pipeline: weights=%s", HOLOPART_WEIGHTS_ROOT)+ load_pipeline(HOLOPART_WEIGHTS_ROOT)+ _pipeline_loaded = True+ log.info("HoloPart pipeline loaded")+++async def _load_pipeline_bg() -> None:+ """Load the pipeline off the request path; retry with backoff before+ latching a failure. See workers/model-anigen's identical function for why+ the first failure is never latched."""+ global _load_error, _load_attempts+ loop = asyncio.get_event_loop()+ attempts = max(1, MODEL_LOAD_ATTEMPTS)+ for attempt in range(1, attempts + 1):+ _load_attempts = attempt+ try:+ await loop.run_in_executor(None, _load_pipeline_sync)+ _load_error = None+ _ready.set()+ return+ except Exception as exc: # noqa: BLE001 - surfaced via /health + task status+ if attempt >= attempts:+ _load_error = safe_error(exc, context="model load")+ log.error("HoloPart pipeline load FAILED after %d attempt(s): %s", attempt, exc)+ return+ delay = min(MODEL_LOAD_RETRY_BASE_S * (2 ** (attempt - 1)), MODEL_LOAD_RETRY_CAP_S)+ log.warning(+ "HoloPart pipeline load attempt %d/%d failed (%s); retrying in %.0fs",+ attempt, attempts, exc, delay,+ )+ await asyncio.sleep(delay)+++@asynccontextmanager+async def lifespan(app: FastAPI):+ global _bucket, _local_storage, _sem, _ready+ if GCS_BUCKET:+ _bucket = storage.Client().bucket(GCS_BUCKET)+ _local_storage = None+ else:+ _bucket = None+ _local_storage = LocalStorage(OUTPUT_DIR)+ _sem = asyncio.Semaphore(MAX_CONCURRENT)+ _ready = asyncio.Event()+ asyncio.create_task(_load_pipeline_bg())+ log.info("Service starting, pipeline loading in background (max_concurrent=%d)", MAX_CONCURRENT)+ yield+++app = FastAPI(title="model-holopart", lifespan=lifespan)+++def _require_api_key(authorization: str) -> None:+ try:+ require_api_key(authorization, API_KEY)+ except PermissionError as exc:+ raise HTTPException(status_code=401, detail=str(exc)) from exc+++class MeshSourceError(ValueError):+ """A caller-supplied mesh could not be read: see workers/segment's+ identical rejection shape for FBX / unsafe URLs."""+++def _source_label(src: str) -> str:+ host = urlsplit(src).hostname+ return host or src[:60]+++def _url_suffix(url: str) -> str:+ return Path(url.split("?")[0].split("#")[0]).suffix.lower()+++def _is_fbx(data: bytes) -> bool:+ if data.startswith(FBX_BINARY_MAGIC):+ return True+ return data[:64].lstrip(b"\xef\xbb\xbf").lstrip().startswith(FBX_ASCII_HEADER)+++def _fetch_mesh_bytes(url: str) -> bytes:+ label = _source_label(url)+ last_exc: Optional[BaseException] = None+ for attempt in range(1, FETCH_ATTEMPTS + 1):+ try:+ return fetch_remote_bytes(url, timeout=MESH_FETCH_TIMEOUT_S, max_bytes=MAX_MESH_BYTES)+ except UnsafeUrlError as exc:+ raise MeshSourceError(f"refused to fetch mesh ({label}): {exc}") from exc+ except httpx.HTTPStatusError as exc:+ raise MeshSourceError(f"mesh source {label} returned HTTP {exc.response.status_code}") from exc+ except httpx.HTTPError as exc:+ last_exc = exc+ if attempt < FETCH_ATTEMPTS:+ log.warning(+ "mesh fetch attempt %d/%d for %s failed (%s); retrying",+ attempt, FETCH_ATTEMPTS, label, type(exc).__name__,+ )+ time.sleep(min(2 ** attempt, 5))+ except ValueError as exc:+ raise MeshSourceError(f"mesh source {label} rejected: {exc}") from exc+ raise MeshSourceError(+ f"mesh source {label} unreachable after {FETCH_ATTEMPTS} attempts "+ f"({type(last_exc).__name__}); check the URL is publicly readable"+ )+++def _load_part_scene(mesh_url: str):+ import trimesh++ data = _fetch_mesh_bytes(mesh_url)+ if _url_suffix(mesh_url) == ".fbx" or _is_fbx(data):+ raise MeshSourceError(UNSUPPORTED_FORMAT_MESSAGE)+ suffix = _url_suffix(mesh_url)+ if suffix not in SUPPORTED_INPUT_FORMATS:+ suffix = ".glb"+ try:+ data, suffix = decode_if_meshopt(data, suffix)+ except ValueError as exc:+ raise MeshSourceError(str(exc)) from exc+ import io++ scene = trimesh.load(io.BytesIO(data), file_type=suffix.lstrip("."))+ if not isinstance(scene, trimesh.Scene) or len(scene.geometry) < 2:+ raise MeshSourceError(UNSUPPORTED_FORMAT_MESSAGE)+ return scene+++def _task_blob(task_id: str):+ if _bucket is None:+ raise RuntimeError("GCS task storage is not active")+ return _bucket.blob(f"tasks/{task_id}.json")+++def _using_gcs() -> bool:+ return bool(GCS_BUCKET)+++def _get_local_storage() -> LocalStorage:+ global _local_storage+ if _local_storage is None:+ _local_storage = LocalStorage(OUTPUT_DIR)+ return _local_storage+++async def _update_task(task_id: str, **fields) -> dict:+ lock = _task_locks.setdefault(task_id, asyncio.Lock())+ async with lock:+ task = _tasks.setdefault(task_id, {"task_id": task_id})+ task.update(fields)+ task["updated_at"] = time.time()+ snapshot = task.copy()+ loop = asyncio.get_event_loop()+ if _using_gcs():+ payload = json.dumps(snapshot)+ await loop.run_in_executor(+ None,+ lambda: _task_blob(task_id).upload_from_string(+ payload, content_type="application/json"+ ),+ )+ else:+ await loop.run_in_executor(None, _get_local_storage().write_task, snapshot)+ return task+++_TERMINAL_STATUSES = frozenset({"done", "failed"})+++async def _resolve_task(task_id: str) -> dict:+ task = _tasks.get(task_id)+ if task is not None and task.get("status") in _TERMINAL_STATUSES:+ return task+ loop = asyncio.get_event_loop()+ try:+ if _using_gcs():+ data = await loop.run_in_executor(None, _task_blob(task_id).download_as_bytes)+ task = json.loads(data)+ else:+ task = await loop.run_in_executor(None, _get_local_storage().read_task, task_id)+ except (NotFound, FileNotFoundError, ValueError):+ if task is not None:+ return task+ raise HTTPException(status_code=404, detail="task not found")+ except Exception as exc:+ raise HTTPException(status_code=502, detail=safe_error(exc, context="task lookup")) from exc+ if task.get("status") in _TERMINAL_STATUSES:+ _tasks[task_id] = task+ return task+++def _generate_sync(scene, output_glb: Path, seed: int) -> dict:+ return run_holopart(scene, output_glb, holopart_weights_dir=HOLOPART_WEIGHTS_ROOT, seed=seed)+++def _release_gpu_memory() -> None:+ gc.collect()+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ torch.cuda.ipc_collect()+++async def _run_job(task_id: str, mesh_url: str, seed: int | None) -> None:+ if _load_error:+ await _update_task(task_id, status="failed", error=f"pipeline unavailable: {_load_error}")+ return+ try:+ await asyncio.wait_for(_ready.wait(), timeout=900)+ except asyncio.TimeoutError:+ await _update_task(task_id, status="failed", error="pipeline not ready (model load timed out)")+ return+ if _load_error:+ await _update_task(task_id, status="failed", error=f"pipeline unavailable: {_load_error}")+ return++ async with _sem:+ await _update_task(task_id, status="running")+ loop = asyncio.get_event_loop()+ t0 = time.time()+ import tempfile++ try:+ scene = await loop.run_in_executor(None, _load_part_scene, mesh_url)+ with tempfile.TemporaryDirectory() as tmp:+ output_glb = Path(tmp) / "completed.glb"+ result = await loop.run_in_executor(+ None, _generate_sync, scene, output_glb, seed if seed is not None else 42+ )+ glb_bytes = result["glb_bytes"]++ if _using_gcs():+ blob_name = f"raw-meshes/holopart/{task_id}.glb"+ blob = _bucket.blob(blob_name)+ await loop.run_in_executor(+ None, lambda: blob.upload_from_string(glb_bytes, content_type="model/gltf-binary")+ )+ result_fields = {"result_gcs_url": f"https://storage.googleapis.com/{GCS_BUCKET}/{blob_name}"}+ result_label = result_fields["result_gcs_url"]+ else:+ result_path = await loop.run_in_executor(+ None, _get_local_storage().write_result, task_id, glb_bytes+ )+ result_fields = {"result_url": f"/results/{task_id}.glb", "result_path": str(result_path)}+ result_label = str(result_path)++ elapsed = time.time() - t0+ await _update_task(+ task_id,+ status="done",+ **result_fields,+ parts=result["parts"],+ part_count=result["part_count"],+ bytes=len(glb_bytes),+ elapsed_ms=int(elapsed * 1000),+ )+ log.info(+ "[%s] done in %.1fs (parts=%d) %d bytes -> %s",+ task_id, elapsed, result["part_count"], len(glb_bytes), result_label,+ )+ except MeshSourceError as exc:+ log.warning("[%s] mesh source rejected: %s", task_id, exc)+ await _update_task(task_id, status="failed", error=str(exc), elapsed_ms=int((time.time() - t0) * 1000))+ except PartCompletionError as exc:+ log.error("[%s] HoloPart produced no usable parts: %s", task_id, exc)+ await _update_task(task_id, status="failed", error=str(exc), elapsed_ms=int((time.time() - t0) * 1000))+ except Exception as exc:+ await _update_task(+ task_id, status="failed", error=safe_error(exc, context=f"[{task_id}] inference"),+ elapsed_ms=int((time.time() - t0) * 1000),+ )+ finally:+ await loop.run_in_executor(None, _release_gpu_memory)+++class InferRequest(BaseModel):+ mesh_url: str = Field(..., min_length=1, max_length=4096)+ job_id: str | None = None+ seed: int | None = None++[email protected]("/infer", status_code=202)+async def infer(body: InferRequest, background_tasks: BackgroundTasks, authorization: str = Header(...)) -> dict:+ _require_api_key(authorization)+ if _load_error:+ raise HTTPException(status_code=503, detail=f"pipeline unavailable: {_load_error}")+ task_id = str(uuid.uuid4())+ await _update_task(task_id, status="queued", model="holopart")+ background_tasks.add_task(_run_job, task_id, body.mesh_url, body.seed)+ return {"task_id": task_id, "status": "queued"}++[email protected]("/tasks/{task_id}")+async def get_task(task_id: str, authorization: str = Header(...)) -> dict:+ _require_api_key(authorization)+ return await _resolve_task(task_id)++[email protected]("/results/{task_id}.glb")+async def get_result(task_id: str, authorization: str = Header(...)):+ _require_api_key(authorization)+ if _using_gcs():+ raise HTTPException(status_code=404, detail="local result storage is not active")+ try:+ path = _get_local_storage().result_path(task_id)+ except ValueError as exc:+ raise HTTPException(status_code=404, detail="result not found") from exc+ if not path.is_file():+ raise HTTPException(status_code=404, detail="result not found")+ return FileResponse(path, media_type="model/gltf-binary", filename=path.name)++[email protected]("/")+async def root() -> dict:+ return {+ "service": "model-holopart",+ "model": "holopart",+ "ready": bool(_ready and _ready.is_set()),+ "endpoints": ["POST /infer", "GET /tasks/{task_id}", "GET /results/{task_id}.glb", "GET /health"],+ }++[email protected]("/health")+async def health(response: Response) -> dict:+ dead = _load_error is not None+ if dead:+ response.status_code = 503+ return {+ "ok": not dead,+ "model": "holopart",+ "gpu_available": torch.cuda.is_available(),+ "gpu_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,+ "pipeline_loaded": _pipeline_loaded,+ "ready": bool(_ready and _ready.is_set()),+ "load_error": _load_error,+ "load_attempts": _load_attempts,+ "output_backend": "gcs" if _using_gcs() else "local",+ "output_dir": None if _using_gcs() else str(_get_local_storage().output_dir),+ }diff --git a/workers/model-holopart/pipeline.py b/workers/model-holopart/pipeline.pynew file mode 100644index 000000000..2351420bc--- /dev/null+++ b/workers/model-holopart/pipeline.py@@ -0,0 +1,263 @@+"""+HoloPart (VAST-AI-Research/HoloPart, MIT) in-process pipeline wrapper: an+already part-segmented mesh in, a mesh with every part's occluded geometry+completed out. Unlike PartCrafter (which generates parts from a 2D image),+HoloPart takes a 3D mesh that is ALREADY split into named part geometries+(a trimesh.Scene GLB, one node per part, exactly the shape workers/segment+emits) and fills in the geometry each part is missing where it touches or+is hidden behind its neighbors.++Upstream's own script (scripts/inference_holopart.py) reads this straight+from `trimesh.Scene.geometry.items()` and, critically, writes+`part_mesh.name = part_id_list[i]` on every output part: node names survive+the round trip with no extra work from this wrapper, which is exactly what+work order 070's Task 3 (keep node names stable across sources) requires.+This wrapper mirrors workers/model-anigen's importable-pipeline pattern: load+the model once behind a lock, call it directly, no subprocess.+"""+from __future__ import annotations++import logging+import threading+from pathlib import Path++log = logging.getLogger("model-holopart.pipeline")++_lock = threading.Lock()+_pipeline = None++NUM_SURFACE_SAMPLES = 20480+PART_NORMALIZE_SCALE = 0.7+MIN_PARTS = 2+MAX_PARTS = 64+++def load_pipeline(holopart_weights_dir: str, device: str = "cuda"):+ """Load (once) and return the HoloPartPipeline singleton.++ Import is deferred to call time, not module top: this lets+ test_app_contract.py exercise request validation and output handling+ without HoloPart's checkout or its GPU dependencies installed.+ """+ global _pipeline+ with _lock:+ if _pipeline is None:+ import torch+ from holopart.pipelines.pipeline_holopart import HoloPartPipeline++ log.info("loading HoloPart pipeline: weights=%s", holopart_weights_dir)+ pipeline = HoloPartPipeline.from_pretrained(holopart_weights_dir)+ pipeline = pipeline.to(device, torch.float16)+ _pipeline = pipeline+ return _pipeline+++class PartCompletionError(RuntimeError):+ """HoloPart ran but produced nothing usable: the input GLB had fewer+ than two named part geometries, or every decoded part came back empty.+ Not a transient failure; the caller's mesh needs re-segmenting first."""+++def _prepare_batch(scene, device: str):+ """Build HoloPart's per-part + whole-object conditioning tensors from a+ trimesh.Scene whose geometries are already the named parts, mirroring+ upstream's own prepare_data() for the .glb branch exactly (including its+ per-part local normalization), minus its argparse/CLI plumbing."""+ import numpy as np+ import torch+ import trimesh+ from torch_cluster import nearest++ part_name_list: list[str] = []+ part_pcd_list = []+ whole_cond_list = []+ part_cond_list = []+ part_local_cond_list = []+ part_center_list = []+ part_scale_list = []++ geometries = list(scene.geometry.items())+ if len(geometries) < MIN_PARTS:+ raise PartCompletionError(+ f"input mesh has {len(geometries)} named part geometr{'y' if len(geometries) == 1 else 'ies'}; "+ f"HoloPart needs at least {MIN_PARTS} (segment the mesh into parts first)"+ )++ for i, (name, part_mesh) in enumerate(geometries):+ part_surface_points, face_idx = part_mesh.sample(NUM_SURFACE_SAMPLES, return_index=True)+ part_surface_normals = part_mesh.face_normals[face_idx]+ part_pcd = np.concatenate([part_surface_points, np.ones_like(part_surface_points[:, :1]) * i], axis=-1)+ part_pcd_list.append(part_pcd)++ part_surface_points_t = torch.FloatTensor(part_surface_points)+ part_surface_normals_t = torch.FloatTensor(part_surface_normals)+ part_cond = torch.cat([part_surface_points_t, part_surface_normals_t], dim=-1)+ part_local_cond = part_cond.clone()+ part_cond_max = part_local_cond[:, :3].max(dim=0)[0]+ part_cond_min = part_local_cond[:, :3].min(dim=0)[0]+ part_center_new = (part_cond_max + part_cond_min) / 2+ part_local_cond[:, :3] = part_local_cond[:, :3] - part_center_new+ part_scale_new = (part_local_cond[:, :3].abs().max() / (0.95 * PART_NORMALIZE_SCALE)).item()+ part_local_cond[:, :3] = part_local_cond[:, :3] / part_scale_new+ part_cond_list.append(part_cond)+ part_local_cond_list.append(part_local_cond)+ part_name_list.append(name)+ part_center_list.append(part_center_new)+ part_scale_list.append(part_scale_new)++ part_pcd = np.concatenate(part_pcd_list, axis=0)+ part_pcd_t = torch.FloatTensor(part_pcd).to(device)+ whole_mesh = scene.dump(concatenate=True)+ whole_surface_points, face_idx = whole_mesh.sample(NUM_SURFACE_SAMPLES, return_index=True)+ whole_surface_normals = whole_mesh.face_normals[face_idx]+ whole_surface_points_t = torch.FloatTensor(whole_surface_points)+ whole_surface_normals_t = torch.FloatTensor(whole_surface_normals)+ whole_surface_points_device = whole_surface_points_t.to(device)+ nearest_idx = nearest(whole_surface_points_device, part_pcd_t[:, :3])+ nearest_part = part_pcd_t[nearest_idx][:, 3].cpu()+ for i in range(len(part_cond_list)):+ surface_points_part_mask = (nearest_part == i).float()+ whole_cond = torch.cat(+ [whole_surface_points_t, whole_surface_normals_t, surface_points_part_mask[..., None]], dim=-1+ )+ whole_cond_list.append(whole_cond)++ return {+ "whole_cond": torch.stack(whole_cond_list, dim=0).to(device),+ "part_cond": torch.stack(part_cond_list, dim=0).to(device),+ "part_local_cond": torch.stack(part_local_cond_list, dim=0).to(device),+ "part_id_list": part_name_list,+ "part_center_list": part_center_list,+ "part_scale_list": part_scale_list,+ }+++def _simplify_mesh(mesh, n_faces: int):+ import pymeshlab+ import trimesh++ pm_mesh = pymeshlab.Mesh(mesh.vertices, mesh.faces)+ ms = pymeshlab.MeshSet()+ ms.add_mesh(pm_mesh)+ ms.meshing_merge_close_vertices()+ ms.meshing_decimation_quadric_edge_collapse(targetfacenum=n_faces)+ simplified = ms.current_mesh()+ return trimesh.Trimesh(vertices=simplified.vertex_matrix(), faces=simplified.face_matrix())+++def _golden_ratio_color(index: int) -> str:+ """Same visual convention as workers/segment/segment_core.py and+ workers/model-partcrafter/pipeline.py: a distinct, evenly-spread hue per+ part. Upstream's own script tints with `np.random.rand`; this wrapper+ uses the platform's deterministic scheme instead, so a completed part+ keeps the same color a caller already associated with that part id."""+ import colorsys++ golden_ratio_conjugate = 0.618033988749895+ hue = (index * golden_ratio_conjugate) % 1.0+ r, g, b = colorsys.hsv_to_rgb(hue, 0.55, 0.92)+ return "#%02x%02x%02x" % (round(r * 255), round(g * 255), round(b * 255))+++def run_holopart(+ scene,+ output_glb: Path,+ *,+ holopart_weights_dir: str,+ seed: int = 42,+ num_inference_steps: int = 50,+ guidance_scale: float = 3.5,+ batch_size: int = 8,+ simplify_target_faces: int = 10000,+ device: str = "cuda",+) -> dict:+ """Run HoloPart end to end on an already part-segmented trimesh.Scene,+ writing a single named-node GLB (part names preserved verbatim from the+ input) to `output_glb`, and return a parts manifest shaped like+ workers/segment's (so both sources render through the same Parts Studio /+ Restyle Studio UI)."""+ import numpy as np+ import torch+ import trimesh++ from holopart.inference_utils import flash_extract_geometry++ pipeline = load_pipeline(holopart_weights_dir, device=device)+ batch = _prepare_batch(scene, device=device)++ part_surface = batch["part_cond"]+ whole_surface = batch["whole_cond"]+ part_local_surface = batch["part_local_cond"]+ part_id_list = batch["part_id_list"]+ part_center_list = batch["part_center_list"]+ part_scale_list = batch["part_scale_list"]++ latent_list = []+ with torch.no_grad():+ for i in range(0, len(part_surface), batch_size):+ meshes_latent = pipeline(+ part_surface=part_surface[i : i + batch_size],+ whole_surface=whole_surface[i : i + batch_size],+ part_local_surface=part_local_surface[i : i + batch_size],+ num_inference_steps=num_inference_steps,+ guidance_scale=guidance_scale,+ generator=torch.Generator(device=device).manual_seed(seed),+ output_type="latent",+ ).samples+ latent_list.append(meshes_latent)+ meshes_latent = torch.cat(latent_list, dim=0)+ pipeline.vae.set_flash_decoder()++ usable_parts = []+ out_scene = trimesh.Scene()+ parts_manifest: list[dict] = []+ for i, mesh_latent in enumerate(meshes_latent):+ output = flash_extract_geometry(+ mesh_latent.unsqueeze(0),+ pipeline.vae,+ bounds=(-1.05, -1.05, -1.05, 1.05, 1.05, 1.05),+ octree_depth=9,+ num_chunks=10000,+ )+ if not output or output[0] is None:+ continue+ meshes = [trimesh.Trimesh(mesh_v_f[0].astype(np.float32), mesh_v_f[1]) for mesh_v_f in output]+ part_mesh = trimesh.util.concatenate(meshes)+ if len(part_mesh.vertices) == 0:+ continue+ part_mesh = _simplify_mesh(part_mesh, simplify_target_faces)+ part_mesh.apply_scale(part_scale_list[i])+ part_mesh.apply_translation(part_center_list[i])++ node_name = part_id_list[i]+ color = _golden_ratio_color(i)+ rgba = trimesh.visual.color.hex_to_rgba(color)+ part_mesh.visual = trimesh.visual.ColorVisuals(+ part_mesh, vertex_colors=np.tile(rgba, (len(part_mesh.vertices), 1))+ )+ out_scene.add_geometry(part_mesh, node_name=node_name, geom_name=node_name)+ bbox = part_mesh.bounds+ parts_manifest.append({+ "id": node_name,+ "name": node_name,+ "region": node_name,+ "face_count": int(len(part_mesh.faces)),+ "vertex_count": int(len(part_mesh.vertices)),+ "bbox": {"min": bbox[0].tolist(), "max": bbox[1].tolist()},+ "centroid": part_mesh.centroid.tolist(),+ "volume": float(part_mesh.volume) if part_mesh.is_volume else 0.0,+ "color": color,+ })+ usable_parts.append(node_name)++ if not usable_parts:+ raise PartCompletionError("HoloPart completed no usable parts from this mesh")++ output_glb.parent.mkdir(parents=True, exist_ok=True)+ out_scene.export(output_glb, file_type="glb")++ return {+ "glb_bytes": output_glb.read_bytes(),+ "parts": parts_manifest,+ "part_count": len(usable_parts),+ }diff --git a/workers/model-holopart/requirements.txt b/workers/model-holopart/requirements.txtnew file mode 100644index 000000000..57f87f99f--- /dev/null+++ b/workers/model-holopart/requirements.txt@@ -0,0 +1,12 @@+# This worker's own FastAPI/GCS layer. torch/torchvision and HoloPart's own+# requirements.txt install in their own Dockerfile layers BEFORE this file+# (HoloPart ships no pip-installable package; it is imported straight from+# its vendored checkout on PYTHONPATH).+trimesh>=4.4.0,<5++fastapi>=0.115,<1+uvicorn[standard]>=0.32,<1+pydantic>=2.0.0+httpx>=0.27.0++google-cloud-storage==2.18.0diff --git a/workers/model-holopart/stage-assets.sh b/workers/model-holopart/stage-assets.shnew file mode 100644index 000000000..dc444bbf8--- /dev/null+++ b/workers/model-holopart/stage-assets.sh@@ -0,0 +1,31 @@+#!/usr/bin/env bash+# Stage HoloPart's weights into the model-weights bucket. Run once (from any+# machine with gcloud + python3 + huggingface_hub) before the first deploy,+# and again only when bumping HOLOPART_COMMIT in the Dockerfile.+#+# bash workers/model-holopart/stage-assets.sh+#+# Layout produced under gs://three-ws-model-weights/holopart/:+# HoloPart/... diffusion weights (VAST-AI/HoloPart)+#+# This is the exact same call upstream's own inference_holopart.py makes at+# runtime when its local weights dir is missing; staging it here ahead of+# time means entrypoint.sh never needs Hugging Face egress in production.+set -euo pipefail++BUCKET="${BUCKET:-gs://three-ws-model-weights/holopart}"+WORK="$(mktemp -d)"+trap 'rm -rf "$WORK"' EXIT++echo "==> HoloPart weights (HF: VAST-AI/HoloPart)"+python3 -c "+from huggingface_hub import snapshot_download+path = snapshot_download(repo_id='VAST-AI/HoloPart', local_dir='$WORK/HoloPart')+print('downloaded to', path)+"++echo "==> Upload to $BUCKET"+gcloud storage cp -r "$WORK/HoloPart" "$BUCKET/HoloPart"++echo "==> Staged. Verify:"+gcloud storage ls -r "$BUCKET" | sed 's/^/ /'diff --git a/workers/model-holopart/storage_backend.py b/workers/model-holopart/storage_backend.pynew file mode 100644index 000000000..fcf7fc1a1--- /dev/null+++ b/workers/model-holopart/storage_backend.py@@ -0,0 +1,63 @@+"""Storage primitives for the standalone TRELLIS container.++Production persists tasks and results in GCS because requests can move between+Cloud Run instances. An NGC user runs one container with a mounted output+volume, so the same service needs a local backend with atomic writes and strict+task-id validation.+"""++from __future__ import annotations++import json+import os+import uuid+from pathlib import Path+++def normalize_task_id(task_id: str) -> str:+ """Return a canonical UUID or reject a value that could escape the volume."""+ parsed = uuid.UUID(task_id)+ canonical = str(parsed)+ if task_id.lower() != canonical:+ raise ValueError("task_id must be a canonical UUID")+ return canonical+++class LocalStorage:+ """Durable task metadata and GLB results on a mounted local volume."""++ def __init__(self, output_dir: str | os.PathLike[str]):+ self.output_dir = Path(output_dir).expanduser().resolve()+ self.tasks_dir = self.output_dir / "tasks"+ self.results_dir = self.output_dir / "results"+ self.tasks_dir.mkdir(parents=True, exist_ok=True)+ self.results_dir.mkdir(parents=True, exist_ok=True)++ def task_path(self, task_id: str) -> Path:+ return self.tasks_dir / f"{normalize_task_id(task_id)}.json"++ def result_path(self, task_id: str) -> Path:+ return self.results_dir / f"{normalize_task_id(task_id)}.glb"++ @staticmethod+ def _atomic_write(path: Path, data: bytes) -> None:+ temporary = path.with_suffix(path.suffix + ".tmp")+ with temporary.open("wb") as handle:+ handle.write(data)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(path)++ def write_task(self, task: dict) -> Path:+ path = self.task_path(str(task["task_id"]))+ payload = json.dumps(task, separators=(",", ":"), sort_keys=True).encode("utf-8")+ self._atomic_write(path, payload)+ return path++ def read_task(self, task_id: str) -> dict:+ return json.loads(self.task_path(task_id).read_text(encoding="utf-8"))++ def write_result(self, task_id: str, glb_bytes: bytes) -> Path:+ path = self.result_path(task_id)+ self._atomic_write(path, glb_bytes)+ return pathdiff --git a/workers/model-holopart/test_app_contract.py b/workers/model-holopart/test_app_contract.pynew file mode 100644index 000000000..2af297d18--- /dev/null+++ b/workers/model-holopart/test_app_contract.py@@ -0,0 +1,134 @@+"""Smoke test for the served contract, run INSIDE the built image (it needs+torch, FastAPI, trimesh, google-cloud-storage, and the vendored HoloPart tree+on PYTHONPATH):++ docker build -t model-holopart workers/model-holopart+ docker run --rm model-holopart python3 test_app_contract.py++It needs no GPU, no weights, and no GCP credentials: the ASGI lifespan (which+opens the GCS client and starts the model load) is deliberately not run, so+every assertion below is about request handling, not inference.++What it proves:++ 1. The whole app module imports.+ 2. The auth boundary rejects a wrong bearer on both authenticated routes.+ 3. The unauthenticated probes (/ and /health) answer, and a dead pipeline+ says so on both /health and /infer so the caller fails over.+ 4. The mesh loader refuses everything the SSRF guard and the FBX/format+ checks should refuse, including a valid GLB with too few named parts.+ 5. pipeline.py's part-completion guard rejects a scene with fewer than two+ named geometries before any GPU work would start.+"""++from __future__ import annotations++import io+import os+import sys++os.environ.setdefault("API_KEY", "smoke-test-key")+os.environ.setdefault("GCS_BUCKET", "smoke-test-bucket")++KEY = os.environ["API_KEY"]++PASSED = 0+++def check(name: str, condition: bool, detail: str = "") -> None:+ global PASSED+ if not condition:+ print(f"FAIL {name} {detail}")+ sys.exit(1)+ PASSED += 1+ print(f"ok {name}")+++def _single_part_glb_bytes() -> bytes:+ import trimesh++ scene = trimesh.Scene()+ scene.add_geometry(trimesh.creation.box(), node_name="part_01", geom_name="part_01")+ buf = io.BytesIO()+ scene.export(buf, file_type="glb")+ return buf.getvalue()+++def main() -> int:+ import main as app_module+ from fastapi.testclient import TestClient++ client = TestClient(app_module.app)++ bad = {"Authorization": "Bearer wrong-key"}+ res = client.get("/tasks/does-not-exist", headers=bad)+ check("GET /tasks rejects a wrong bearer", res.status_code == 401, f"got {res.status_code}")+ res = client.post("/infer", headers=bad, json={"mesh_url": "https://example.com/a.glb"})+ check("POST /infer rejects a wrong bearer", res.status_code == 401, f"got {res.status_code}")++ res = client.get("/health")+ body = res.json()+ check("GET /health answers 200", res.status_code == 200, f"got {res.status_code}")+ check("health names the model", body.get("model") == "holopart", str(body)[:120])+ res = client.get("/")+ body = res.json()+ check("GET / answers 200 for the keep-warm ping", res.status_code == 200, f"got {res.status_code}")+ check("root names the service", body.get("service") == "model-holopart", str(body)[:120])+ check("root lists the working endpoints", "POST /infer" in body.get("endpoints", []), str(body)[:160])++ app_module._load_error = "internal error (ref deadbeefcafe)"+ try:+ res = client.get("/health")+ body = res.json()+ check("a dead pipeline makes /health answer 503", res.status_code == 503, f"got {res.status_code}")+ check("a dead pipeline makes /health report ok:false", body.get("ok") is False, str(body)[:160])+ check("health surfaces the load error", "deadbeefcafe" in str(body.get("load_error")), str(body)[:160])+ res = client.post(+ "/infer", headers={"Authorization": f"Bearer {KEY}"},+ json={"mesh_url": "https://example.com/a.glb"},+ )+ check("a dead pipeline makes /infer refuse with 503", res.status_code == 503, f"got {res.status_code}")+ finally:+ app_module._load_error = None+ res = client.get("/health")+ check("a live pipeline still answers /health 200", res.status_code == 200, f"got {res.status_code}")+ check("a live pipeline reports ok:true", res.json().get("ok") is True, str(res.json())[:160])++ good = {"Authorization": f"Bearer {KEY}"}+ res = client.post("/infer", headers=good, json={})+ check("POST /infer requires mesh_url", res.status_code == 422, f"got {res.status_code}")++ for label, src in [+ ("cleartext http", "http://example.com/a.glb"),+ ("bare path", "/etc/passwd"),+ ("loopback https", "https://127.0.0.1/a.glb"),+ ("cloud metadata", "https://169.254.169.254/latest/meta-data/"),+ ]:+ try:+ app_module._fetch_mesh_bytes(src)+ check(f"refuses {label}", False, "no exception raised")+ except app_module.MeshSourceError:+ check(f"refuses {label}", True)+ except Exception as exc: # noqa: BLE001 - must be the caller-facing class+ check(f"refuses {label}", False, f"raised {type(exc).__name__}: {exc}")++ from pipeline import MIN_PARTS, PartCompletionError, _prepare_batch++ try:+ import trimesh++ single_part_scene = trimesh.load(io.BytesIO(_single_part_glb_bytes()), file_type="glb")+ _prepare_batch(single_part_scene, device="cpu")+ check("rejects a scene with fewer than MIN_PARTS geometries", False, "no exception raised")+ except PartCompletionError as exc:+ check("rejects a scene with fewer than MIN_PARTS geometries", True, str(exc))+ except ModuleNotFoundError as exc:+ print(f"skip part-count guard (torch_cluster unavailable in this environment): {exc}")+ check("MIN_PARTS matches the documented floor", MIN_PARTS == 2, str(MIN_PARTS))++ print(f"\n{PASSED} contract assertions passed")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/workers/model-holopart/test_gltf_meshopt.py b/workers/model-holopart/test_gltf_meshopt.pynew file mode 100644index 000000000..2d2e9d502--- /dev/null+++ b/workers/model-holopart/test_gltf_meshopt.py@@ -0,0 +1,141 @@+"""Tests for gltf_meshopt.py, the shared EXT_meshopt_compression decoder.++Vendored byte-identical into every worker that ships gltf_meshopt.py, exactly+like the module itself, and run as a Docker build gate in each of them:++ python3 test_gltf_meshopt.py++The round-trip section needs the `gltfpack` binary (present in every worker+image, pinned in the Dockerfile). Without it the suite still runs and pins the+readable failure a caller gets instead, so a laptop run stays green.+"""++from __future__ import annotations++import io+import os+import shutil+import subprocess+import sys+import tempfile+from pathlib import Path++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))++import trimesh # noqa: E402++import gltf_meshopt # noqa: E402++PASS = 0+++def check(name: str, condition: bool, detail: str = "") -> None:+ global PASS+ if not condition:+ print(f"FAIL {name} {detail}")+ sys.exit(1)+ PASS += 1+ print(f"ok {name}")+++plain_glb = trimesh.Scene({"a": trimesh.creation.icosphere(subdivisions=2)}).export(file_type="glb")++# ── document parsing ────────────────────────────────────────────────────────────++doc = gltf_meshopt.gltf_document(plain_glb, ".glb")+check("a glb's json chunk parses", isinstance(doc, dict) and "asset" in doc, str(type(doc)))+check(+ "a gltf json payload parses",+ isinstance(gltf_meshopt.gltf_document(b'{"asset":{"version":"2.0"}}', ".gltf"), dict),+)+for payload, suffix, label in (+ (b"ply\nformat ascii 1.0\n", ".ply", "a ply"),+ (b"not json at all", ".gltf", "malformed gltf json"),+ (b"glTF" + b"\x00" * 4, ".glb", "a truncated glb"),+ (b"solid mesh\n", ".stl", "an stl"),+):+ check(f"{label} yields no gltf document", gltf_meshopt.gltf_document(payload, suffix) is None)++# ── extension detection ─────────────────────────────────────────────────────────++check("a plain glb needs no decode", not gltf_meshopt.uses_meshopt(doc))+check("an empty document needs no decode", not gltf_meshopt.uses_meshopt(None))+for declared in ("extensionsRequired", "extensionsUsed"):+ check(+ f"{declared} announcing {gltf_meshopt.MESHOPT_EXTENSION} needs a decode",+ gltf_meshopt.uses_meshopt({declared: [gltf_meshopt.MESHOPT_EXTENSION]}),+ )+check(+ "an unrelated extension needs no decode",+ not gltf_meshopt.uses_meshopt({"extensionsUsed": ["KHR_materials_unlit"]}),+)+check(+ "an uncompressed payload passes through decode_if_meshopt untouched",+ gltf_meshopt.decode_if_meshopt(plain_glb, ".glb") == (plain_glb, ".glb"),+)++# ── the real decode ─────────────────────────────────────────────────────────────++if shutil.which(gltf_meshopt.GLTFPACK_BIN):+ with tempfile.TemporaryDirectory() as packdir:+ raw = Path(packdir) / "raw.glb"+ packed = Path(packdir) / "packed.glb"+ raw.write_bytes(plain_glb)+ subprocess.run(+ [gltf_meshopt.GLTFPACK_BIN, "-i", str(raw), "-o", str(packed), "-cc"],+ capture_output=True,+ check=True,+ timeout=gltf_meshopt.GLTFPACK_TIMEOUT_S,+ )+ packed_bytes = packed.read_bytes()++ check(+ "a gltfpack-compressed glb is detected",+ gltf_meshopt.uses_meshopt(gltf_meshopt.gltf_document(packed_bytes, ".glb")),+ )++ # The bug this module exists for: trimesh dies on the compressed asset's+ # fallback buffer, which is why every worker loading one failed.+ try:+ trimesh.load(io.BytesIO(packed_bytes), file_type="glb", force="mesh", process=False)+ raw_load_failed = False+ except Exception: # noqa: BLE001: the exact trimesh error is not the contract+ raw_load_failed = True+ check("trimesh alone cannot read a meshopt glb", raw_load_failed)++ decoded, suffix = gltf_meshopt.decode_if_meshopt(packed_bytes, ".glb")+ check("the decode returns a glb", suffix == ".glb" and decoded[:4] == b"glTF", suffix)+ check("the decode is not a passthrough", decoded != packed_bytes)+ check(+ "the decoded asset no longer declares the extension",+ not gltf_meshopt.uses_meshopt(gltf_meshopt.gltf_document(decoded, ".glb")),+ )++ mesh = trimesh.load(io.BytesIO(decoded), file_type="glb", force="mesh", process=False)+ check(+ "the decoded asset loads as real geometry",+ len(mesh.faces) > 0 and len(mesh.vertices) > 0,+ f"faces={len(mesh.faces)}",+ )+ source = trimesh.load(io.BytesIO(plain_glb), file_type="glb", force="mesh", process=False)+ check(+ "the decoded geometry matches the source face count",+ len(mesh.faces) == len(source.faces),+ f"{len(mesh.faces)} vs {len(source.faces)}",+ )+else:+ original_bin = gltf_meshopt.GLTFPACK_BIN+ try:+ gltf_meshopt.GLTFPACK_BIN = "gltfpack-not-installed"+ gltf_meshopt.transcode_meshopt(plain_glb, ".glb")+ check("a missing gltfpack is a readable error", False, "no ValueError raised")+ except ValueError as exc:+ check(+ "a missing gltfpack is a readable error",+ gltf_meshopt.MESHOPT_EXTENSION in str(exc),+ str(exc),+ )+ finally:+ gltf_meshopt.GLTFPACK_BIN = original_bin++print(f"\n{PASS} checks passed")diff --git a/workers/model-holopart/worker_security.py b/workers/model-holopart/worker_security.pynew file mode 100644index 000000000..802e1c32d--- /dev/null+++ b/workers/model-holopart/worker_security.py@@ -0,0 +1,320 @@+"""+Shared security helpers for the three.ws GPU/CPU workers.++This module is intentionally dependency-light (stdlib only) so it can be+COPYed verbatim into every worker's Docker build context. The same canonical+file lives in each worker directory; keep them byte-identical when editing.++It provides three primitives that every worker needs:++ 1. ``require_api_key`` - constant-time bearer-token check (timing-safe).+ 2. ``fetch_remote_bytes`` - SSRF-hardened HTTP(S) fetch of caller-supplied+ URLs: https-only, DNS resolution with private/loopback/link-local/metadata+ IP rejection, per-hop redirect re-validation, and a bounded response size.+ 3. ``safe_error`` - turns an exception into an opaque, correlation-id-tagged+ message safe to persist/return to callers, after logging the full+ traceback server-side only.+"""++from __future__ import annotations++import hmac+import ipaddress+import logging+import asyncio+import socket+import time+import uuid+from typing import Iterable, Optional+from urllib.parse import urlsplit++import httpx++log = logging.getLogger("worker_security")++# ── auth ────────────────────────────────────────────────────────────────────────+++def require_api_key(authorization: Optional[str], api_key: str) -> None:+ """Validate a ``Bearer <token>`` header in constant time.++ Raises ``PermissionError`` on any failure so callers can translate it into+ the framework's 401 (FastAPI workers map this to HTTPException(401)).+ """+ # Fail closed on an empty configured key: with api_key == "" a bare+ # "Bearer " header would otherwise strip to "" and compare equal.+ if not api_key:+ raise PermissionError("worker api key not configured")+ if not authorization or not authorization.startswith("Bearer "):+ raise PermissionError("missing bearer token")+ token = authorization[len("Bearer "):].strip()+ if not token:+ raise PermissionError("missing bearer token")+ # Compare bytes: hmac.compare_digest is timing-safe, and on str operands it+ # raises TypeError for non-ASCII input, which would surface as a 500.+ if not hmac.compare_digest(token.encode("utf-8"), api_key.encode("utf-8")):+ raise PermissionError("invalid api key")+++# ── SSRF-hardened fetch ─────────────────────────────────────────────────────────++# Default cap on a fetched response. Avatar selfies / driving audio / raw meshes+# are comfortably under this; anything larger is almost certainly abuse.+DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB+_MAX_REDIRECTS = 4++# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)++# Hosts that enforce a User-Agent policy refuse httpx's default+# "python-httpx/<version>": upload.wikimedia.org answers it with 403 Forbidden+# and serves the same image to a descriptive agent (measured 2026-10-09, when+# model-triposg failed every job whose reference photo was a Wikimedia link).+# Callers may still pass their own User-Agent; it replaces this one.+DEFAULT_USER_AGENT = "three.ws-worker/1.0 (+https://three.ws)"+++def _fetch_headers(headers: Optional[dict]) -> dict:+ """Caller headers over the default User-Agent, matched case-insensitively."""+ merged = dict(headers or {})+ if not any(str(k).lower() == "user-agent" for k in merged):+ merged["User-Agent"] = DEFAULT_USER_AGENT+ return merged+++class UnsafeUrlError(ValueError):+ """Raised when a URL targets a disallowed scheme or a private/internal IP."""+++def _is_blocked_ip(ip: ipaddress._BaseAddress) -> bool:+ """Reject any address that could reach internal infrastructure.++ Covers loopback, RFC1918 private ranges, link-local (incl. the cloud+ metadata server at 169.254.169.254), unique-local IPv6 (fc00::/7),+ multicast, reserved, unspecified, and any other non-global address (CGNAT+ 100.64.0.0/10 included). IPv4-mapped IPv6 addresses+ are unwrapped first so a mapped private v4 can't slip through.+ """+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:+ ip = ip.ipv4_mapped+ return (+ ip.is_private+ or ip.is_loopback+ or ip.is_link_local+ or ip.is_multicast+ or ip.is_reserved+ or ip.is_unspecified+ # Anything not globally routable, which also covers the CGNAT shared+ # space 100.64.0.0/10 that is_private does not flag.+ or not ip.is_global+ )+++def _resolve_safe_host(host: str) -> None:+ """DNS-resolve ``host`` and raise UnsafeUrlError if ANY address is internal.++ Resolving and checking every returned address closes the gap where a name+ resolves to both a public and a private address.+ """+ if not host:+ raise UnsafeUrlError("missing host")+ try:+ infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)+ except socket.gaierror as exc:+ raise UnsafeUrlError(f"could not resolve host: {host}") from exc++ addrs = {info[4][0] for info in infos}+ if not addrs:+ raise UnsafeUrlError(f"no addresses for host: {host}")+ for addr in addrs:+ ip = ipaddress.ip_address(addr)+ if _is_blocked_ip(ip):+ raise UnsafeUrlError(f"host resolves to a disallowed address: {host}")+++def assert_safe_url(url: str, *, allow_http: bool = False) -> str:+ """Validate scheme + resolved IPs for a single URL. Returns it unchanged.++ https is required by default; ``allow_http`` exists only for callers that+ legitimately need cleartext (not used by the workers).+ """+ parts = urlsplit(url)+ scheme = parts.scheme.lower()+ allowed = ("https", "http") if allow_http else ("https",)+ if scheme not in allowed:+ raise UnsafeUrlError(f"scheme not allowed: {scheme or '(none)'}")+ if not parts.hostname:+ raise UnsafeUrlError("missing host")+ _resolve_safe_host(parts.hostname)+ return url+++def fetch_remote_bytes(+ url: str,+ *,+ timeout: float = 30.0,+ max_bytes: int = DEFAULT_MAX_BYTES,+ allow_http: bool = False,+ headers: Optional[dict] = None,+) -> bytes:+ """Fetch ``url`` safely and return its body bytes.++ Hardening applied:+ * https-only (unless ``allow_http``).+ * every URL - including each redirect target - is DNS-resolved and+ rejected if it maps to a private/loopback/link-local/metadata address.+ * redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is+ re-validated; httpx's own redirect following is disabled.+ * the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.++ Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /+ ``ValueError`` for transport or size failures.+ """+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:+ current = assert_safe_url(url, allow_http=allow_http)+ with httpx.Client(follow_redirects=False, timeout=timeout) as client:+ for _ in range(_MAX_REDIRECTS + 1):+ with client.stream("GET", current, headers=_fetch_headers(headers)) as resp:+ if resp.is_redirect:+ location = resp.headers.get("location")+ if not location:+ raise UnsafeUrlError("redirect without Location header")+ current = assert_safe_url(+ str(resp.url.join(location)), allow_http=allow_http+ )+ continue+ resp.raise_for_status()+ declared = resp.headers.get("content-length")+ if declared is not None and declared.isdigit() and int(declared) > max_bytes:+ raise ValueError(f"response too large: {declared} bytes")+ chunks: list[bytes] = []+ total = 0+ for chunk in resp.iter_bytes(65536):+ total += len(chunk)+ if total > max_bytes:+ raise ValueError(f"response exceeded {max_bytes} bytes")+ chunks.append(chunk)+ return b"".join(chunks)+ raise UnsafeUrlError(f"too many redirects (>{_MAX_REDIRECTS})")+++async def fetch_remote_bytes_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int = DEFAULT_MAX_BYTES,+ allow_http: bool = False,+ headers: Optional[dict] = None,+) -> bytes:+ """Async variant of ``fetch_remote_bytes`` using a caller-provided client.++ The provided ``client`` MUST be constructed with ``follow_redirects=False``+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant.+ """+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:+ current = assert_safe_url(url, allow_http=allow_http)+ for _ in range(_MAX_REDIRECTS + 1):+ async with client.stream("GET", current, headers=_fetch_headers(headers)) as resp:+ if resp.is_redirect:+ location = resp.headers.get("location")+ if not location:+ raise UnsafeUrlError("redirect without Location header")+ current = assert_safe_url(+ str(resp.url.join(location)), allow_http=allow_http+ )+ continue+ resp.raise_for_status()+ declared = resp.headers.get("content-length")+ if declared is not None and declared.isdigit() and int(declared) > max_bytes:+ raise ValueError(f"response too large: {declared} bytes")+ chunks: list[bytes] = []+ total = 0+ async for chunk in resp.aiter_bytes(65536):+ total += len(chunk)+ if total > max_bytes:+ raise ValueError(f"response exceeded {max_bytes} bytes")+ chunks.append(chunk)+ return b"".join(chunks)+ raise UnsafeUrlError(f"too many redirects (>{_MAX_REDIRECTS})")+++# ── opaque error handling ───────────────────────────────────────────────────────+++def safe_error(exc: BaseException, *, context: str = "") -> str:+ """Log the full traceback server-side and return an opaque client message.++ Never leak ``traceback.format_exc()`` (absolute paths, library versions,+ internal hostnames, fragments of upstream responses) to callers. A short+ correlation id is embedded in both the log line and the returned string so+ operators can join a client-reported failure to the server log.+ """+ correlation_id = uuid.uuid4().hex[:12]+ label = f"{context} " if context else ""+ log.exception("%serror [correlation_id=%s]", label, correlation_id)+ return f"internal error (ref {correlation_id})"diff --git a/workers/model-partcrafter/Dockerfile b/workers/model-partcrafter/Dockerfilenew file mode 100644index 000000000..b32ae78b9--- /dev/null+++ b/workers/model-partcrafter/Dockerfile@@ -0,0 +1,61 @@+# PartCrafter (wgsxm/PartCrafter, MIT, LICENSE read 2026-10-11): one+# reference image to a part-separated 3D asset, N named part meshes decoded+# in a single diffusion-transformer pass. Built on a TripoSG-derived+# backbone; its own requirements.txt pins torch 2.4.1+cu121.+FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04++ENV PYTHONDONTWRITEBYTECODE=1 \+ PYTHONUNBUFFERED=1 \+ PIP_NO_CACHE_DIR=1 \+ DEBIAN_FRONTEND=noninteractive \+ PYTHONPATH=/app/partcrafter-src \+ HF_HUB_OFFLINE=1 \+ TRANSFORMERS_OFFLINE=1 \+ HF_HUB_DISABLE_TELEMETRY=1 \+ DO_NOT_TRACK=1++RUN apt-get update && apt-get install -y --no-install-recommends \+ python3.10 python3.10-dev python3-pip \+ git curl build-essential ninja-build \+ libgl1 libglib2.0-0 libgomp1 \+ && rm -rf /var/lib/apt/lists/* \+ && ln -sf /usr/bin/python3.10 /usr/bin/python3 \+ && ln -sf /usr/bin/python3.10 /usr/bin/python \+ && pip install --upgrade pip setuptools wheel++# 1. torch, its own cached layer (every extension below links against it).+RUN pip install torch==2.4.1 torchvision==0.19.1 --index-url https://download.pytorch.org/whl/cu121++# 2. PartCrafter, pinned to the commit read on 2026-10-11+# (https://github.com/wgsxm/PartCrafter/commit/3d773bf02fad51c7ab31a5615573fec93b287b30).+ARG PARTCRAFTER_COMMIT=3d773bf02fad51c7ab31a5615573fec93b287b30+RUN git clone https://github.com/wgsxm/PartCrafter.git /app/partcrafter-src \+ && git -C /app/partcrafter-src checkout ${PARTCRAFTER_COMMIT} \+ && rm -rf /app/partcrafter-src/.git++# 3. PartCrafter's own requirements (minus torch/torchvision, installed above).+RUN grep -viE '^(torch|torchvision)([=<> ]|$)' /app/partcrafter-src/requirements.txt > /tmp/partcrafter-reqs.txt \+ && pip install -r /tmp/partcrafter-reqs.txt++WORKDIR /app+COPY requirements.txt /app/requirements.txt+RUN pip install -r /app/requirements.txt++COPY worker_security.py storage_backend.py pipeline.py main.py entrypoint.sh /app/+COPY test_app_contract.py /app/+RUN chmod +x /app/entrypoint.sh++# Build-time guard: served-contract tests need no GPU and no checkpoints, so+# they run here and the image does not ship if the import chain or request+# handling broke.+RUN python3 /app/test_app_contract.py++# Weights are staged in GCS (stage-assets.sh) and mounted at /weights by+# Cloud Run; entrypoint.sh copies PartCrafter + RMBG-1.4 into+# PARTCRAFTER_WEIGHTS_ROOT, the literal relative layout pipeline.py's+# from_pretrained calls use, so inference needs no Hugging Face egress at+# runtime.++EXPOSE 8080++CMD ["/app/entrypoint.sh"]diff --git a/workers/model-partcrafter/README.md b/workers/model-partcrafter/README.mdnew file mode 100644index 000000000..bf0f23375--- /dev/null+++ b/workers/model-partcrafter/README.md@@ -0,0 +1,124 @@+# model-partcrafter: editable-parts generation worker++Every other Forge image-to-3D lane (TRELLIS.2, Pixal3D, Hunyuan3D, TripoSG,+AniGen, ...) hands back one undivided mesh. This worker, built on+[PartCrafter](https://github.com/wgsxm/PartCrafter) (wgsxm, MIT, LICENSE read+2026-10-11, NeurIPS 2025), decodes a reference photo straight into N+individually named part meshes in a single diffusion-transformer pass, so the+result is editable (recolor, swap, regenerate one part) from the moment it is+generated. This is the Forge "Parts" lane for work order 070; `model-holopart`+is the companion worker that completes occluded geometry on meshes that were+part-segmented after the fact (by `workers/segment`), rather than generated+with parts already separated.++## How it works++`PartCrafterPipeline` (imported directly, in-process: PartCrafter is a real+Python package on its own checkout, not a subprocess CLI) conditions a shared+TripoSG-derived diffusion transformer on the SAME background-removed+reference image once per requested part+(`attention_kwargs={"num_parts": N}`), and decodes all N meshes in one+forward pass. See `pipeline.py` for the exact call.++Upstream's own script (`scripts/inference_partcrafter.py`) exports each part+as its own `part_{i:02}.glb` file plus a separately merged `object.glb`. This+worker instead builds ONE GLB scene with named nodes directly+(`part_01`, `part_02`, ... one-indexed, two digits) and tints each part with+the same golden-ratio hue stepping `workers/segment` uses+(`hue = (index * 0.618033988749895) % 1.0`), so a PartCrafter generation and a+post-hoc segmentation are indistinguishable in shape to every downstream+consumer: Restyle Studio's part-editing UI, Parts Studio, and the node-name+stability contract work order 078 ("keep the rig") depends on.++## API++```+POST /infer { images, num_parts (2-16), seed?, job_id? }+ -> 202 { task_id, status: "queued" } (Bearer API_KEY)+GET /tasks/:id -> { task_id, status, result_gcs_url?, result_url?,+ parts?, part_count?, views_used?, error? }+GET /results/:id.glb -> generated GLB (local OUTPUT_DIR mode only)+GET /health -> { ok, model, gpu_available, gpu_name, load_error }+```++Same task shape as every other Forge image lane (`POST /infer` + poll+`GET /tasks/:id`), so the Forge router dispatches to it exactly as it does+TRELLIS.2 or AniGen. PartCrafter conditions on one image; when several are+sent the first is used and the task record reports `views_used: 1`. The+`parts` array on a completed task is shaped identically to+`workers/segment`'s own manifest entries: `{id, name, region, face_count,+vertex_count, bbox, centroid, volume, color}`.++## Deploy++One-time weight staging (under `gs://three-ws-model-weights/partcrafter/`):++```bash+bash workers/model-partcrafter/stage-assets.sh+```++Build + deploy (Cloud Build pins the `three-ws-build@` service account):++```bash+gcloud builds submit --config workers/model-partcrafter/cloudbuild.yaml \+ --substitutions=SHORT_SHA=manual$(date +%s) .+```++Service: `model-partcrafter`, us-central1, 1x L4 (`--no-gpu-zonal-redundancy`),+scale-to-zero by default against the shared per-region L4 quota in+`docs/ops/gcp-credits-plan.md`.++Then point the Forge API at it:++```bash+gcloud run services update three-ws-api --region us-central1 \+ --update-env-vars MODEL_PARTCRAFTER_URL=$(gcloud run services describe \+ model-partcrafter --region us-central1 --format='value(status.url)')+```++`MODEL_PARTCRAFTER_URL` is unset by default, so until that command runs, the+`partcrafter` Forge backend stays inactive (same invariant as every other+self-hosted image lane: it is opt-in only, never a default or a failover+target).++## Local tests (no GPU needed)++```bash+cd workers/model-partcrafter+python3 -m py_compile main.py pipeline.py storage_backend.py worker_security.py test_app_contract.py+```++`test_app_contract.py` additionally proves the served contract (auth+boundary, request validation, SSRF guard, health reporting), but it needs+torch, FastAPI, and the vendored PartCrafter checkout on `PYTHONPATH`, so it+runs inside the built image (`docker build -t model-partcrafter+workers/model-partcrafter`, which runs it as a build-time gate; see the+Dockerfile tail) rather than bare on a dev machine.++## Licensing++PartCrafter is MIT (LICENSE read 2026-10-11). RMBG-1.4 (briaai) is used+unmodified for background removal only, per its own HuggingFace model card+terms.++## Environment++| Var | Default | Meaning |+|---|---|---|+| `API_KEY` | (secret) | shared bearer secret (`avatar-reconstruction-key`) |+| `GCS_BUCKET` | (optional) | output bucket for part-separated GLBs; local `OUTPUT_DIR` storage when unset |+| `OUTPUT_DIR` | `/output` | local task/result volume when `GCS_BUCKET` is unset |+| `PARTCRAFTER_WEIGHTS_ROOT` | `/app/pretrained_weights` | local root `entrypoint.sh` stages `PartCrafter/` and `RMBG-1.4/` under |+| `MAX_CONCURRENT` | `1` | parallel inference jobs per instance |+| `WEIGHTS_ROOT` | `/weights/partcrafter` | staged weights, read by `entrypoint.sh` |+| `MODEL_LOAD_ATTEMPTS` / `MODEL_LOAD_RETRY_BASE_S` / `MODEL_LOAD_RETRY_CAP_S` | `4` / `15` / `120` | model-load retry budget |+| `IMAGE_FETCH_TIMEOUT_S` | `30` | per-attempt image fetch timeout |++## Failure modes++| Reported `error` | Cause | Fix |+|---|---|---|+| `unsupported image source: ...` / `image source ... rejected` | bad or unreachable `images[0]` entry | fix the URL; rejected before touching the GPU |+| `PartCrafter decoded no usable parts from this image` | every decoded part was empty | retry with a clearer, more separable subject, or a lower `num_parts` |+| `pipeline unavailable: ...` / `pipeline not ready` | weight load failed or is still in progress | check `/health`'s `load_error`; retry once the instance reports ready |+| `internal error (ref <id>)` | anything unexpected | grep the service logs for the correlation id |diff --git a/workers/model-partcrafter/cloudbuild.yaml b/workers/model-partcrafter/cloudbuild.yamlnew file mode 100644index 000000000..a3dd0b63d--- /dev/null+++ b/workers/model-partcrafter/cloudbuild.yaml@@ -0,0 +1,80 @@+# Build + deploy the PartCrafter "editable parts" worker.+# Manual submit (SHORT_SHA is only auto-set by triggers):+# gcloud builds submit --config workers/model-partcrafter/cloudbuild.yaml \+# --substitutions=SHORT_SHA=manual$(date +%s) .+serviceAccount: projects/aerial-vehicle-466722-p5/serviceAccounts/[email protected]++substitutions:+ _REGION: us-central1+ _RUN_SA: avatar-reconstruction-sa@aerial-vehicle-466722-p5.iam.gserviceaccount.com+ _SERVICE: model-partcrafter+ _REPO: model-partcrafter+ _GCS_BUCKET: three-ws-avatar-reconstructions+ _WEIGHTS_BUCKET: three-ws-model-weights+ # Scale-to-zero by default, same reasoning as workers/model-anigen: the+ # us-central1 L4 grant is shared, and a warm floor here is a quota decision,+ # not a default. Raise with `gcloud run services update model-partcrafter+ # --min-instances=1` once the grant allows.+ _MIN_INSTANCES: "0"+ _MAX_INSTANCES: "2"++steps:+ - id: build+ name: gcr.io/cloud-builders/docker+ dir: workers/model-partcrafter+ args:+ - build+ - --tag+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA+ - --tag+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest+ - --cache-from+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest+ - .++ - id: push+ name: gcr.io/cloud-builders/docker+ args:+ - push+ - --all-tags+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server++ - id: deploy+ name: gcr.io/google.com/cloudsdktool/cloud-sdk:slim+ entrypoint: gcloud+ args:+ - run+ - deploy+ - ${_SERVICE}+ - --image=${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA+ - --region=${_REGION}+ - --service-account=${_RUN_SA}+ - --platform=managed+ - --port=8080+ - --gpu=1+ - --gpu-type=nvidia-l4+ - --no-gpu-zonal-redundancy+ - --no-cpu-throttling+ - --cpu=4+ - --memory=16Gi+ - --min-instances=${_MIN_INSTANCES}+ - --max-instances=${_MAX_INSTANCES}+ - --timeout=900+ - --startup-probe=tcpSocket.port=8080,initialDelaySeconds=0,periodSeconds=60,timeoutSeconds=60,failureThreshold=15+ - --cpu-boost+ - --allow-unauthenticated+ - --add-volume=name=weights,type=cloud-storage,bucket=${_WEIGHTS_BUCKET}+ - --add-volume-mount=volume=weights,mount-path=/weights+ - --set-env-vars=GCS_BUCKET=${_GCS_BUCKET},MAX_CONCURRENT=1,WEIGHTS_ROOT=/weights/partcrafter+ - --set-secrets=API_KEY=avatar-reconstruction-key:latest+ - --liveness-probe=httpGet.path=/health,initialDelaySeconds=60,periodSeconds=60,timeoutSeconds=10,failureThreshold=3++images:+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:$SHORT_SHA+ - ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/server:latest++options:+ machineType: E2_HIGHCPU_32+ logging: CLOUD_LOGGING_ONLY++timeout: 3600sdiff --git a/workers/model-partcrafter/entrypoint.sh b/workers/model-partcrafter/entrypoint.shnew file mode 100644index 000000000..4ce1ddf93--- /dev/null+++ b/workers/model-partcrafter/entrypoint.sh@@ -0,0 +1,27 @@+#!/bin/sh+# Wire staged PartCrafter + RMBG weights into the local paths pipeline.py+# loads from (PARTCRAFTER_WEIGHTS_ROOT/PartCrafter, PARTCRAFTER_WEIGHTS_ROOT/RMBG-1.4),+# then serve.+#+# Checkpoints are COPIED from the FUSE-mounted bucket to local disk rather+# than symlinked, same reasoning as workers/model-anigen: torch.load over+# gcsfuse does slow random reads, and a sequential cp of a few GB is fast and+# deterministic.+set -eu++WEIGHTS_ROOT="${WEIGHTS_ROOT:-/weights/partcrafter}"+PARTCRAFTER_WEIGHTS_ROOT="${PARTCRAFTER_WEIGHTS_ROOT:-/app/pretrained_weights}"++if [ ! -e "$WEIGHTS_ROOT/PartCrafter" ] || [ ! -e "$WEIGHTS_ROOT/RMBG-1.4" ]; then+ echo "fatal: expected staged weights at $WEIGHTS_ROOT/PartCrafter and $WEIGHTS_ROOT/RMBG-1.4 (run workers/model-partcrafter/stage-assets.sh)" >&2+ exit 1+fi++mkdir -p "$PARTCRAFTER_WEIGHTS_ROOT"+rm -rf "$PARTCRAFTER_WEIGHTS_ROOT/PartCrafter" "$PARTCRAFTER_WEIGHTS_ROOT/RMBG-1.4"+echo "staging $WEIGHTS_ROOT/PartCrafter -> $PARTCRAFTER_WEIGHTS_ROOT/PartCrafter"+cp -r "$WEIGHTS_ROOT/PartCrafter" "$PARTCRAFTER_WEIGHTS_ROOT/PartCrafter"+echo "staging $WEIGHTS_ROOT/RMBG-1.4 -> $PARTCRAFTER_WEIGHTS_ROOT/RMBG-1.4"+cp -r "$WEIGHTS_ROOT/RMBG-1.4" "$PARTCRAFTER_WEIGHTS_ROOT/RMBG-1.4"++exec uvicorn main:app --host 0.0.0.0 --port 8080 --workers 1 --timeout-keep-alive 300diff --git a/workers/model-partcrafter/main.py b/workers/model-partcrafter/main.pynew file mode 100644index 000000000..426b243b4--- /dev/null+++ b/workers/model-partcrafter/main.py@@ -0,0 +1,444 @@+"""+PartCrafter inference service: one image to a part-separated 3D asset+(wgsxm/PartCrafter, MIT; structured 3D generation that decodes several+individually editable part meshes from one RGB image in a single pass).++This is the Forge "Parts" lane (work order 070): where every other image+lane returns one undivided mesh, this worker returns a GLB whose nodes are+already named parts (`part_01`, `part_02`, ...), ready for the same Restyle+Studio part-select flow that workers/segment's post-hoc split feeds, and+sized by the caller's requested part count instead of a learned guess.++API contract (consumed by the Forge router, same task shape as the other+image-to-3D workers):+ POST /infer { images: [data-uri|url, ...], num_parts: int (2-16),+ seed?: int, job_id?: str }+ -> 202 { task_id, status: "queued" }++ PartCrafter conditions on ONE image, repeated once per requested part; when+ several images are sent the first is used and the task record reports+ views_used: 1.++ GET /tasks/:id -> { task_id, status, result_gcs_url?, result_url?,+ parts?, part_count?, error? }++ GET /results/:id.glb -> generated GLB when OUTPUT_DIR local storage is active++ GET /health -> { ok, model, gpu_available, load_error, load_attempts }.+ 503 with ok:false once the model load has spent its+ retry budget, so a dead instance is drained.++ GET / -> { service, model, ready, endpoints }. Unauthenticated+ descriptor for the platform's warmth probe.++Weights: gs://three-ws-model-weights/partcrafter (workers/model-partcrafter/stage-assets.sh).++Environment variables (README.md carries the full table):+ API_KEY shared bearer secret+ GCS_BUCKET optional Cloud Storage bucket for output meshes+ OUTPUT_DIR local task and result volume when GCS_BUCKET is unset+ PARTCRAFTER_WEIGHTS_ROOT local checkpoint root entrypoint.sh stages into (default: /app/pretrained_weights)+ MAX_CONCURRENT max parallel inferences (default: 1)+ IMAGE_FETCH_TIMEOUT_S per-attempt image fetch timeout (default: 30)+ MODEL_LOAD_ATTEMPTS model-load attempts before the error latches (default: 4)+ MODEL_LOAD_RETRY_BASE_S / MODEL_LOAD_RETRY_CAP_S load-retry backoff bounds+"""++from __future__ import annotations++import asyncio+import base64+import gc+import json+import logging+import os+import time+import uuid+from contextlib import asynccontextmanager+from pathlib import Path+from typing import Optional+from urllib.parse import urlsplit++import httpx+import torch+from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Response+from fastapi.responses import FileResponse+from google.api_core.exceptions import NotFound+from google.cloud import storage+from pydantic import BaseModel, Field++from pipeline import MAX_NUM_PARTS, PartGenerationError, run_partcrafter+from storage_backend import LocalStorage+from worker_security import (+ UnsafeUrlError,+ fetch_remote_bytes,+ require_api_key,+ safe_error,+)++logging.basicConfig(+ level=logging.INFO,+ format="%(asctime)s %(levelname)s %(name)s %(message)s",+)+log = logging.getLogger("model-partcrafter")++API_KEY = os.environ["API_KEY"]+GCS_BUCKET = os.environ.get("GCS_BUCKET", "").strip()+OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "/output")+PARTCRAFTER_WEIGHTS_ROOT = os.environ.get("PARTCRAFTER_WEIGHTS_ROOT", "/app/pretrained_weights")+PARTCRAFTER_DIR = os.path.join(PARTCRAFTER_WEIGHTS_ROOT, "PartCrafter")+RMBG_DIR = os.path.join(PARTCRAFTER_WEIGHTS_ROOT, "RMBG-1.4")+MAX_CONCURRENT = int(os.environ.get("MAX_CONCURRENT", "1"))+MODEL_LOAD_ATTEMPTS = int(os.environ.get("MODEL_LOAD_ATTEMPTS", "4"))+MODEL_LOAD_RETRY_BASE_S = float(os.environ.get("MODEL_LOAD_RETRY_BASE_S", "15"))+MODEL_LOAD_RETRY_CAP_S = float(os.environ.get("MODEL_LOAD_RETRY_CAP_S", "120"))+IMAGE_FETCH_TIMEOUT_S = float(os.environ.get("IMAGE_FETCH_TIMEOUT_S", "30"))+FETCH_ATTEMPTS = 3++_bucket: Optional[storage.Bucket] = None+_local_storage: Optional[LocalStorage] = None+_sem: Optional[asyncio.Semaphore] = None+_ready: Optional[asyncio.Event] = None+_load_error: Optional[str] = None+_load_attempts = 0+_pipeline_loaded = False+# In-memory cache only, same invariant as workers/model-trellis2: the durable+# source of truth is the tasks/{task_id}.json blob in GCS.+_tasks: dict[str, dict] = {}+_task_locks: dict[str, asyncio.Lock] = {}+++def _load_pipeline_sync() -> None:+ global _pipeline_loaded+ from pipeline import load_pipeline++ log.info("Loading PartCrafter pipeline: weights=%s rmbg=%s", PARTCRAFTER_DIR, RMBG_DIR)+ load_pipeline(PARTCRAFTER_DIR, RMBG_DIR)+ _pipeline_loaded = True+ log.info("PartCrafter pipeline loaded")+++async def _load_pipeline_bg() -> None:+ """Load the pipeline off the request path; retry with backoff before+ latching a failure. See workers/model-trellis2's identical function for+ why the first failure is never latched: a transient blip must not turn a+ warm, minScale-held instance into a permanently dead one."""+ global _load_error, _load_attempts+ loop = asyncio.get_event_loop()+ attempts = max(1, MODEL_LOAD_ATTEMPTS)+ for attempt in range(1, attempts + 1):+ _load_attempts = attempt+ try:+ await loop.run_in_executor(None, _load_pipeline_sync)+ _load_error = None+ _ready.set()+ return+ except Exception as exc: # noqa: BLE001 - surfaced via /health + task status+ if attempt >= attempts:+ _load_error = safe_error(exc, context="model load")+ log.error("PartCrafter pipeline load FAILED after %d attempt(s): %s", attempt, exc)+ return+ delay = min(MODEL_LOAD_RETRY_BASE_S * (2 ** (attempt - 1)), MODEL_LOAD_RETRY_CAP_S)+ log.warning(+ "PartCrafter pipeline load attempt %d/%d failed (%s); retrying in %.0fs",+ attempt, attempts, exc, delay,+ )+ await asyncio.sleep(delay)+++@asynccontextmanager+async def lifespan(app: FastAPI):+ global _bucket, _local_storage, _sem, _ready+ if GCS_BUCKET:+ _bucket = storage.Client().bucket(GCS_BUCKET)+ _local_storage = None+ else:+ _bucket = None+ _local_storage = LocalStorage(OUTPUT_DIR)+ _sem = asyncio.Semaphore(MAX_CONCURRENT)+ _ready = asyncio.Event()+ asyncio.create_task(_load_pipeline_bg())+ log.info("Service starting, pipeline loading in background (max_concurrent=%d)", MAX_CONCURRENT)+ yield+++app = FastAPI(title="model-partcrafter", lifespan=lifespan)+++def _require_api_key(authorization: str) -> None:+ try:+ require_api_key(authorization, API_KEY)+ except PermissionError as exc:+ raise HTTPException(status_code=401, detail=str(exc)) from exc+++class ImageSourceError(ValueError):+ """A caller-supplied image could not be read: see workers/model-trellis2."""+++def _source_label(src: str) -> str:+ if src.startswith("data:image"):+ return "inline data uri"+ host = urlsplit(src).hostname+ return host or src[:60]+++def _fetch_image_bytes(src: str) -> bytes:+ label = _source_label(src)+ last_exc: Optional[BaseException] = None+ for attempt in range(1, FETCH_ATTEMPTS + 1):+ try:+ return fetch_remote_bytes(src, timeout=IMAGE_FETCH_TIMEOUT_S)+ except UnsafeUrlError as exc:+ raise ImageSourceError(f"refused to fetch image source ({label}): {exc}") from exc+ except httpx.HTTPStatusError as exc:+ raise ImageSourceError(+ f"image source {label} returned HTTP {exc.response.status_code}"+ ) from exc+ except httpx.HTTPError as exc:+ last_exc = exc+ if attempt < FETCH_ATTEMPTS:+ log.warning(+ "image fetch attempt %d/%d for %s failed (%s); retrying",+ attempt, FETCH_ATTEMPTS, label, type(exc).__name__,+ )+ time.sleep(min(2 ** attempt, 5))+ except ValueError as exc:+ raise ImageSourceError(f"image source {label} rejected: {exc}") from exc+ raise ImageSourceError(+ f"image source {label} unreachable after {FETCH_ATTEMPTS} attempts "+ f"({type(last_exc).__name__}); check the URL is publicly readable"+ )+++def _decode_image_bytes(src: str) -> bytes:+ if src.startswith("data:image"):+ b64 = src.split(",", 1)[1]+ try:+ return base64.b64decode(b64)+ except Exception as exc: # noqa: BLE001 - caller's own payload+ raise ImageSourceError(f"inline data uri is not valid base64: {exc}") from exc+ if src.startswith("https://"):+ return _fetch_image_bytes(src)+ raise ImageSourceError(f"unsupported image source: {src[:60]}")+++def _task_blob(task_id: str):+ if _bucket is None:+ raise RuntimeError("GCS task storage is not active")+ return _bucket.blob(f"tasks/{task_id}.json")+++def _using_gcs() -> bool:+ return bool(GCS_BUCKET)+++def _get_local_storage() -> LocalStorage:+ global _local_storage+ if _local_storage is None:+ _local_storage = LocalStorage(OUTPUT_DIR)+ return _local_storage+++async def _update_task(task_id: str, **fields) -> dict:+ lock = _task_locks.setdefault(task_id, asyncio.Lock())+ async with lock:+ task = _tasks.setdefault(task_id, {"task_id": task_id})+ task.update(fields)+ task["updated_at"] = time.time()+ snapshot = task.copy()+ loop = asyncio.get_event_loop()+ if _using_gcs():+ payload = json.dumps(snapshot)+ await loop.run_in_executor(+ None,+ lambda: _task_blob(task_id).upload_from_string(+ payload, content_type="application/json"+ ),+ )+ else:+ await loop.run_in_executor(None, _get_local_storage().write_task, snapshot)+ return task+++_TERMINAL_STATUSES = frozenset({"done", "failed"})+++async def _resolve_task(task_id: str) -> dict:+ task = _tasks.get(task_id)+ if task is not None and task.get("status") in _TERMINAL_STATUSES:+ return task+ loop = asyncio.get_event_loop()+ try:+ if _using_gcs():+ data = await loop.run_in_executor(None, _task_blob(task_id).download_as_bytes)+ task = json.loads(data)+ else:+ task = await loop.run_in_executor(None, _get_local_storage().read_task, task_id)+ except (NotFound, FileNotFoundError, ValueError):+ if task is not None:+ return task+ raise HTTPException(status_code=404, detail="task not found")+ except Exception as exc:+ raise HTTPException(status_code=502, detail=safe_error(exc, context="task lookup")) from exc+ if task.get("status") in _TERMINAL_STATUSES:+ _tasks[task_id] = task+ return task+++def _generate_sync(image_bytes: bytes, output_glb: Path, num_parts: int, seed: int) -> dict:+ return run_partcrafter(+ image_bytes,+ output_glb,+ partcrafter_weights_dir=PARTCRAFTER_DIR,+ rmbg_weights_dir=RMBG_DIR,+ num_parts=num_parts,+ seed=seed,+ )+++def _release_gpu_memory() -> None:+ gc.collect()+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ torch.cuda.ipc_collect()+++async def _run_job(task_id: str, images: list[str], num_parts: int, seed: int | None) -> None:+ if _load_error:+ await _update_task(task_id, status="failed", error=f"pipeline unavailable: {_load_error}")+ return+ try:+ await asyncio.wait_for(_ready.wait(), timeout=900)+ except asyncio.TimeoutError:+ await _update_task(task_id, status="failed", error="pipeline not ready (model load timed out)")+ return+ if _load_error:+ await _update_task(task_id, status="failed", error=f"pipeline unavailable: {_load_error}")+ return++ async with _sem:+ await _update_task(task_id, status="running")+ loop = asyncio.get_event_loop()+ t0 = time.time()+ import tempfile++ try:+ image_bytes = await loop.run_in_executor(None, _decode_image_bytes, images[0])+ with tempfile.TemporaryDirectory() as tmp:+ output_glb = Path(tmp) / "parts.glb"+ result = await loop.run_in_executor(+ None, _generate_sync, image_bytes, output_glb, num_parts, seed if seed is not None else 0+ )+ glb_bytes = result["glb_bytes"]++ if _using_gcs():+ blob_name = f"raw-meshes/partcrafter/{task_id}.glb"+ blob = _bucket.blob(blob_name)+ await loop.run_in_executor(+ None, lambda: blob.upload_from_string(glb_bytes, content_type="model/gltf-binary")+ )+ result_fields = {"result_gcs_url": f"https://storage.googleapis.com/{GCS_BUCKET}/{blob_name}"}+ result_label = result_fields["result_gcs_url"]+ else:+ result_path = await loop.run_in_executor(+ None, _get_local_storage().write_result, task_id, glb_bytes+ )+ result_fields = {"result_url": f"/results/{task_id}.glb", "result_path": str(result_path)}+ result_label = str(result_path)++ elapsed = time.time() - t0+ await _update_task(+ task_id,+ status="done",+ **result_fields,+ parts=result["parts"],+ part_count=result["part_count"],+ views_used=1,+ bytes=len(glb_bytes),+ elapsed_ms=int(elapsed * 1000),+ )+ log.info(+ "[%s] done in %.1fs (parts=%d) %d bytes -> %s",+ task_id, elapsed, result["part_count"], len(glb_bytes), result_label,+ )+ except ImageSourceError as exc:+ log.warning("[%s] image source rejected: %s", task_id, exc)+ await _update_task(task_id, status="failed", error=str(exc), elapsed_ms=int((time.time() - t0) * 1000))+ except PartGenerationError as exc:+ log.error("[%s] PartCrafter produced no usable parts: %s", task_id, exc)+ await _update_task(task_id, status="failed", error=str(exc), elapsed_ms=int((time.time() - t0) * 1000))+ except Exception as exc:+ await _update_task(+ task_id, status="failed", error=safe_error(exc, context=f"[{task_id}] inference"),+ elapsed_ms=int((time.time() - t0) * 1000),+ )+ finally:+ await loop.run_in_executor(None, _release_gpu_memory)+++class InferRequest(BaseModel):+ images: list[str] = Field(..., min_length=1, max_length=6)+ num_parts: int = Field(..., ge=2, le=MAX_NUM_PARTS)+ job_id: str | None = None+ seed: int | None = None++[email protected]("/infer", status_code=202)+async def infer(body: InferRequest, background_tasks: BackgroundTasks, authorization: str = Header(...)) -> dict:+ _require_api_key(authorization)+ if _load_error:+ raise HTTPException(status_code=503, detail=f"pipeline unavailable: {_load_error}")+ task_id = str(uuid.uuid4())+ await _update_task(task_id, status="queued", model="partcrafter")+ background_tasks.add_task(_run_job, task_id, body.images, body.num_parts, body.seed)+ return {"task_id": task_id, "status": "queued"}++[email protected]("/tasks/{task_id}")+async def get_task(task_id: str, authorization: str = Header(...)) -> dict:+ _require_api_key(authorization)+ return await _resolve_task(task_id)++[email protected]("/results/{task_id}.glb")+async def get_result(task_id: str, authorization: str = Header(...)):+ _require_api_key(authorization)+ if _using_gcs():+ raise HTTPException(status_code=404, detail="local result storage is not active")+ try:+ path = _get_local_storage().result_path(task_id)+ except ValueError as exc:+ raise HTTPException(status_code=404, detail="result not found") from exc+ if not path.is_file():+ raise HTTPException(status_code=404, detail="result not found")+ return FileResponse(path, media_type="model/gltf-binary", filename=path.name)++[email protected]("/")+async def root() -> dict:+ return {+ "service": "model-partcrafter",+ "model": "partcrafter",+ "ready": bool(_ready and _ready.is_set()),+ "endpoints": ["POST /infer", "GET /tasks/{task_id}", "GET /results/{task_id}.glb", "GET /health"],+ }++[email protected]("/health")+async def health(response: Response) -> dict:+ dead = _load_error is not None+ if dead:+ response.status_code = 503+ return {+ "ok": not dead,+ "model": "partcrafter",+ "gpu_available": torch.cuda.is_available(),+ "gpu_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,+ "pipeline_loaded": _pipeline_loaded,+ "ready": bool(_ready and _ready.is_set()),+ "load_error": _load_error,+ "load_attempts": _load_attempts,+ "output_backend": "gcs" if _using_gcs() else "local",+ "output_dir": None if _using_gcs() else str(_get_local_storage().output_dir),+ }diff --git a/workers/model-partcrafter/pipeline.py b/workers/model-partcrafter/pipeline.pynew file mode 100644index 000000000..986b680c1--- /dev/null+++ b/workers/model-partcrafter/pipeline.py@@ -0,0 +1,162 @@+"""+PartCrafter (wgsxm/PartCrafter, MIT) in-process pipeline wrapper: a single+reference image in, a part-separated GLB out. PartCrafter conditions a shared+TripoSG-derived diffusion transformer on the SAME image once per requested+part and decodes each pass into its own mesh in one shot (its own+`attention_kwargs={"num_parts": N}` call), so this wrapper loads the model+once and calls it directly, mirroring workers/model-anigen's importable-+pipeline pattern rather than workers/rig-nonhuman's subprocess-CLI one.++Upstream's own script (scripts/inference_partcrafter.py) exports each part as+its own `part_{i:02}.glb` plus a merged `object.glb` via+`get_colored_mesh_composition`. This worker instead builds ONE named-node GLB+scene directly (`part_01`, `part_02`, ... one-indexed, two digits), the exact+convention workers/segment emits, so a part-separated generation and a+segmented mesh look identical to every downstream consumer (Restyle Studio,+and the keep-the-rig node-name contract tracked in work order 078).+"""+from __future__ import annotations++import logging+import threading+from pathlib import Path+from typing import Optional++from PIL import Image++log = logging.getLogger("model-partcrafter.pipeline")++_lock = threading.Lock()+_pipeline = None+_rmbg_net = None++MAX_NUM_PARTS = 16+++def load_pipeline(partcrafter_weights_dir: str, rmbg_weights_dir: str, device: str = "cuda"):+ """Load (once) and return the (PartCrafterPipeline, BriaRMBG) pair.++ Import is deferred to call time, not module top: this lets+ test_app_contract.py exercise request validation and output handling+ without PartCrafter's checkout or its GPU dependencies installed.+ """+ global _pipeline, _rmbg_net+ with _lock:+ if _pipeline is None:+ import torch+ from src.models.briarmbg import BriaRMBG+ from src.pipelines.pipeline_partcrafter import PartCrafterPipeline++ log.info(+ "loading PartCrafter pipeline: weights=%s rmbg=%s",+ partcrafter_weights_dir, rmbg_weights_dir,+ )+ pipeline = PartCrafterPipeline.from_pretrained(partcrafter_weights_dir)+ pipeline = pipeline.to("cuda", torch.float16)+ rmbg_net = BriaRMBG.from_pretrained(rmbg_weights_dir).to(device)+ rmbg_net.eval()+ _pipeline = pipeline+ _rmbg_net = rmbg_net+ return _pipeline, _rmbg_net+++class PartGenerationError(RuntimeError):+ """PartCrafter ran but produced nothing usable: every decoded part was+ empty, or fewer parts came back than requested. Not the caller's fault;+ the caller already passed a readable image and a valid part count."""+++def _golden_ratio_color(index: int) -> str:+ """Same visual convention as workers/segment/segment_core.py: a distinct,+ evenly-spread hue per part so segmentation is visible even untextured."""+ import colorsys++ golden_ratio_conjugate = 0.618033988749895+ hue = (index * golden_ratio_conjugate) % 1.0+ r, g, b = colorsys.hsv_to_rgb(hue, 0.55, 0.92)+ return "#%02x%02x%02x" % (round(r * 255), round(g * 255), round(b * 255))+++def run_partcrafter(+ image_bytes: bytes,+ output_glb: Path,+ *,+ partcrafter_weights_dir: str,+ rmbg_weights_dir: str,+ num_parts: int,+ seed: int = 0,+ num_tokens: int = 1024,+ num_inference_steps: int = 50,+ guidance_scale: float = 7.0,+ rmbg: bool = True,+) -> dict:+ """Run PartCrafter end to end on one reference image, writing a single+ named-node GLB (`part_01` .. `part_NN`) to `output_glb`, and return a+ parts manifest shaped like workers/segment's (so both sources render+ through the same Parts Studio / Restyle Studio UI)."""+ import io++ import numpy as np+ import torch+ import trimesh+ from accelerate.utils import set_seed++ from src.utils.image_utils import prepare_image++ pipeline, rmbg_net = load_pipeline(partcrafter_weights_dir, rmbg_weights_dir)+ set_seed(seed)++ image = Image.open(io.BytesIO(image_bytes)).convert("RGB")+ if rmbg:+ img_pil = prepare_image(image, bg_color=np.array([1.0, 1.0, 1.0]), rmbg_net=rmbg_net)+ else:+ img_pil = image++ with torch.no_grad():+ outputs = pipeline(+ image=[img_pil] * num_parts,+ attention_kwargs={"num_parts": num_parts},+ num_tokens=num_tokens,+ generator=torch.Generator(device=pipeline.device).manual_seed(seed),+ num_inference_steps=num_inference_steps,+ guidance_scale=guidance_scale,+ max_num_expanded_coords=int(1e9),+ use_flash_decoder=False,+ ).meshes++ usable = 0+ scene = trimesh.Scene()+ parts_manifest: list[dict] = []+ for i, mesh in enumerate(outputs):+ if mesh is None or len(getattr(mesh, "vertices", [])) == 0:+ continue+ usable += 1+ node_name = f"part_{i + 1:02d}"+ color = _golden_ratio_color(i)+ rgba = trimesh.visual.color.hex_to_rgba(color)+ mesh.visual = trimesh.visual.ColorVisuals(mesh, vertex_colors=np.tile(rgba, (len(mesh.vertices), 1)))+ scene.add_geometry(mesh, node_name=node_name, geom_name=node_name)+ bbox = mesh.bounds+ parts_manifest.append({+ "id": node_name,+ "name": node_name,+ "region": node_name,+ "face_count": int(len(mesh.faces)),+ "vertex_count": int(len(mesh.vertices)),+ "bbox": {"min": bbox[0].tolist(), "max": bbox[1].tolist()},+ "centroid": mesh.centroid.tolist(),+ "volume": float(mesh.volume) if mesh.is_volume else 0.0,+ "color": color,+ })++ if usable == 0:+ raise PartGenerationError("PartCrafter decoded no usable parts from this image")++ output_glb.parent.mkdir(parents=True, exist_ok=True)+ scene.export(output_glb, file_type="glb")++ return {+ "glb_bytes": output_glb.read_bytes(),+ "parts": parts_manifest,+ "part_count": usable,+ }diff --git a/workers/model-partcrafter/requirements.txt b/workers/model-partcrafter/requirements.txtnew file mode 100644index 000000000..54499a48e--- /dev/null+++ b/workers/model-partcrafter/requirements.txt@@ -0,0 +1,12 @@+# This worker's own FastAPI/GCS layer. torch/torchvision, PartCrafter's own+# requirements.txt install in their own Dockerfile layers BEFORE this file+# (PartCrafter ships no pip-installable package; it is imported straight from+# its vendored checkout on PYTHONPATH).+trimesh>=4.4.0,<5++fastapi>=0.115,<1+uvicorn[standard]>=0.32,<1+pydantic>=2.0.0+httpx>=0.27.0++google-cloud-storage==2.18.0diff --git a/workers/model-partcrafter/stage-assets.sh b/workers/model-partcrafter/stage-assets.shnew file mode 100644index 000000000..bdd0a2c6e--- /dev/null+++ b/workers/model-partcrafter/stage-assets.sh@@ -0,0 +1,38 @@+#!/usr/bin/env bash+# Stage PartCrafter's and RMBG's weights into the model-weights bucket. Run+# once (from any machine with gcloud + python3 + huggingface_hub) before the+# first deploy, and again only when bumping PARTCRAFTER_COMMIT in the+# Dockerfile.+#+# bash workers/model-partcrafter/stage-assets.sh+#+# Layout produced under gs://three-ws-model-weights/partcrafter/:+# PartCrafter/... diffusion-transformer weights (wgsxm/PartCrafter)+# RMBG-1.4/... background-removal net (briaai/RMBG-1.4), PartCrafter's+# own preprocessing dependency (scripts/inference_partcrafter.py)+set -euo pipefail++BUCKET="${BUCKET:-gs://three-ws-model-weights/partcrafter}"+WORK="$(mktemp -d)"+trap 'rm -rf "$WORK"' EXIT++echo "==> PartCrafter weights (HF: wgsxm/PartCrafter)"+python3 -c "+from huggingface_hub import snapshot_download+path = snapshot_download(repo_id='wgsxm/PartCrafter', local_dir='$WORK/PartCrafter')+print('downloaded to', path)+"++echo "==> RMBG-1.4 weights (HF: briaai/RMBG-1.4)"+python3 -c "+from huggingface_hub import snapshot_download+path = snapshot_download(repo_id='briaai/RMBG-1.4', local_dir='$WORK/RMBG-1.4')+print('downloaded to', path)+"++echo "==> Upload to $BUCKET"+gcloud storage cp -r "$WORK/PartCrafter" "$BUCKET/PartCrafter"+gcloud storage cp -r "$WORK/RMBG-1.4" "$BUCKET/RMBG-1.4"++echo "==> Staged. Verify:"+gcloud storage ls -r "$BUCKET" | sed 's/^/ /'diff --git a/workers/model-partcrafter/storage_backend.py b/workers/model-partcrafter/storage_backend.pynew file mode 100644index 000000000..fcf7fc1a1--- /dev/null+++ b/workers/model-partcrafter/storage_backend.py@@ -0,0 +1,63 @@+"""Storage primitives for the standalone TRELLIS container.++Production persists tasks and results in GCS because requests can move between+Cloud Run instances. An NGC user runs one container with a mounted output+volume, so the same service needs a local backend with atomic writes and strict+task-id validation.+"""++from __future__ import annotations++import json+import os+import uuid+from pathlib import Path+++def normalize_task_id(task_id: str) -> str:+ """Return a canonical UUID or reject a value that could escape the volume."""+ parsed = uuid.UUID(task_id)+ canonical = str(parsed)+ if task_id.lower() != canonical:+ raise ValueError("task_id must be a canonical UUID")+ return canonical+++class LocalStorage:+ """Durable task metadata and GLB results on a mounted local volume."""++ def __init__(self, output_dir: str | os.PathLike[str]):+ self.output_dir = Path(output_dir).expanduser().resolve()+ self.tasks_dir = self.output_dir / "tasks"+ self.results_dir = self.output_dir / "results"+ self.tasks_dir.mkdir(parents=True, exist_ok=True)+ self.results_dir.mkdir(parents=True, exist_ok=True)++ def task_path(self, task_id: str) -> Path:+ return self.tasks_dir / f"{normalize_task_id(task_id)}.json"++ def result_path(self, task_id: str) -> Path:+ return self.results_dir / f"{normalize_task_id(task_id)}.glb"++ @staticmethod+ def _atomic_write(path: Path, data: bytes) -> None:+ temporary = path.with_suffix(path.suffix + ".tmp")+ with temporary.open("wb") as handle:+ handle.write(data)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(path)++ def write_task(self, task: dict) -> Path:+ path = self.task_path(str(task["task_id"]))+ payload = json.dumps(task, separators=(",", ":"), sort_keys=True).encode("utf-8")+ self._atomic_write(path, payload)+ return path++ def read_task(self, task_id: str) -> dict:+ return json.loads(self.task_path(task_id).read_text(encoding="utf-8"))++ def write_result(self, task_id: str, glb_bytes: bytes) -> Path:+ path = self.result_path(task_id)+ self._atomic_write(path, glb_bytes)+ return pathdiff --git a/workers/model-partcrafter/test_app_contract.py b/workers/model-partcrafter/test_app_contract.pynew file mode 100644index 000000000..99dd5bef0--- /dev/null+++ b/workers/model-partcrafter/test_app_contract.py@@ -0,0 +1,136 @@+"""Smoke test for the served contract, run INSIDE the built image (it needs+torch, FastAPI, PIL, google-cloud-storage, and the vendored PartCrafter tree+on PYTHONPATH):++ docker build -t model-partcrafter workers/model-partcrafter+ docker run --rm model-partcrafter python3 test_app_contract.py++It needs no GPU, no weights, and no GCP credentials: the ASGI lifespan (which+opens the GCS client and starts the model load) is deliberately not run, so+every assertion below is about request handling, not inference.++What it proves:++ 1. The whole app module imports.+ 2. The auth boundary rejects a wrong bearer on both authenticated routes.+ 3. The unauthenticated probes (/ and /health) answer, and a dead pipeline+ says so on both /health and /infer so the caller fails over.+ 4. Request validation rejects an empty/oversized images array and an+ out-of-range num_parts.+ 5. The image decoder refuses everything the SSRF guard should refuse.+"""++from __future__ import annotations++import base64+import io+import os+import sys++os.environ.setdefault("API_KEY", "smoke-test-key")+os.environ.setdefault("GCS_BUCKET", "smoke-test-bucket")++KEY = os.environ["API_KEY"]++PASSED = 0+++def check(name: str, condition: bool, detail: str = "") -> None:+ global PASSED+ if not condition:+ print(f"FAIL {name} {detail}")+ sys.exit(1)+ PASSED += 1+ print(f"ok {name}")+++def png_data_uri(color=(20, 20, 20, 255), size=(24, 24)) -> str:+ from PIL import Image++ img = Image.new("RGBA", size, color)+ buf = io.BytesIO()+ img.save(buf, format="PNG")+ return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()+++def main() -> int:+ import main as app_module+ from fastapi.testclient import TestClient++ client = TestClient(app_module.app)++ bad = {"Authorization": "Bearer wrong-key"}+ res = client.get("/tasks/does-not-exist", headers=bad)+ check("GET /tasks rejects a wrong bearer", res.status_code == 401, f"got {res.status_code}")+ res = client.post("/infer", headers=bad, json={"images": [png_data_uri()], "num_parts": 4})+ check("POST /infer rejects a wrong bearer", res.status_code == 401, f"got {res.status_code}")++ res = client.get("/health")+ body = res.json()+ check("GET /health answers 200", res.status_code == 200, f"got {res.status_code}")+ check("health names the model", body.get("model") == "partcrafter", str(body)[:120])+ res = client.get("/")+ body = res.json()+ check("GET / answers 200 for the keep-warm ping", res.status_code == 200, f"got {res.status_code}")+ check("root names the service", body.get("service") == "model-partcrafter", str(body)[:120])+ check("root lists the working endpoints", "POST /infer" in body.get("endpoints", []), str(body)[:160])++ app_module._load_error = "internal error (ref deadbeefcafe)"+ try:+ res = client.get("/health")+ body = res.json()+ check("a dead pipeline makes /health answer 503", res.status_code == 503, f"got {res.status_code}")+ check("a dead pipeline makes /health report ok:false", body.get("ok") is False, str(body)[:160])+ check("health surfaces the load error", "deadbeefcafe" in str(body.get("load_error")), str(body)[:160])+ res = client.post(+ "/infer", headers={"Authorization": f"Bearer {KEY}"},+ json={"images": [png_data_uri()], "num_parts": 4},+ )+ check("a dead pipeline makes /infer refuse with 503", res.status_code == 503, f"got {res.status_code}")+ finally:+ app_module._load_error = None+ res = client.get("/health")+ check("a live pipeline still answers /health 200", res.status_code == 200, f"got {res.status_code}")+ check("a live pipeline reports ok:true", res.json().get("ok") is True, str(res.json())[:160])++ good = {"Authorization": f"Bearer {KEY}"}+ res = client.post("/infer", headers=good, json={"images": [], "num_parts": 4})+ check("POST /infer rejects an empty images array", res.status_code == 422, f"got {res.status_code}")+ res = client.post("/infer", headers=good, json={"images": [png_data_uri()] * 7, "num_parts": 4})+ check("POST /infer rejects more than six images", res.status_code == 422, f"got {res.status_code}")+ res = client.post("/infer", headers=good, json={"images": [png_data_uri()], "num_parts": 1})+ check("POST /infer rejects num_parts below 2", res.status_code == 422, f"got {res.status_code}")+ res = client.post("/infer", headers=good, json={"images": [png_data_uri()], "num_parts": 17})+ check("POST /infer rejects num_parts above MAX_NUM_PARTS", res.status_code == 422, f"got {res.status_code}")+ res = client.post("/infer", headers=good, json={"images": [png_data_uri()]})+ check("POST /infer requires num_parts", res.status_code == 422, f"got {res.status_code}")++ for label, src in [+ ("cleartext http", "http://example.com/a.png"),+ ("bare path", "/etc/passwd"),+ ("loopback https", "https://127.0.0.1/a.png"),+ ("cloud metadata", "https://169.254.169.254/latest/meta-data/"),+ ("corrupt data uri", "data:image/png;base64,not-base64!!"),+ ]:+ try:+ app_module._decode_image_bytes(src)+ check(f"refuses {label}", False, "no exception raised")+ except app_module.ImageSourceError:+ check(f"refuses {label}", True)+ except Exception as exc: # noqa: BLE001 - must be the caller-facing class+ check(f"refuses {label}", False, f"raised {type(exc).__name__}: {exc}")++ from pipeline import MAX_NUM_PARTS, _golden_ratio_color++ check("MAX_NUM_PARTS matches upstream's own cap", MAX_NUM_PARTS == 16, str(MAX_NUM_PARTS))+ c0 = _golden_ratio_color(0)+ c1 = _golden_ratio_color(1)+ check("golden-ratio colors are distinct per index", c0 != c1, f"{c0} vs {c1}")+ check("golden-ratio colors are hex strings", c0.startswith("#") and len(c0) == 7, c0)++ print(f"\n{PASSED} contract assertions passed")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/workers/model-partcrafter/worker_security.py b/workers/model-partcrafter/worker_security.pynew file mode 100644index 000000000..802e1c32d--- /dev/null+++ b/workers/model-partcrafter/worker_security.py@@ -0,0 +1,320 @@+"""+Shared security helpers for the three.ws GPU/CPU workers.++This module is intentionally dependency-light (stdlib only) so it can be+COPYed verbatim into every worker's Docker build context. The same canonical+file lives in each worker directory; keep them byte-identical when editing.++It provides three primitives that every worker needs:++ 1. ``require_api_key`` - constant-time bearer-token check (timing-safe).+ 2. ``fetch_remote_bytes`` - SSRF-hardened HTTP(S) fetch of caller-supplied+ URLs: https-only, DNS resolution with private/loopback/link-local/metadata+ IP rejection, per-hop redirect re-validation, and a bounded response size.+ 3. ``safe_error`` - turns an exception into an opaque, correlation-id-tagged+ message safe to persist/return to callers, after logging the full+ traceback server-side only.+"""++from __future__ import annotations++import hmac+import ipaddress+import logging+import asyncio+import socket+import time+import uuid+from typing import Iterable, Optional+from urllib.parse import urlsplit++import httpx++log = logging.getLogger("worker_security")++# ── auth ────────────────────────────────────────────────────────────────────────+++def require_api_key(authorization: Optional[str], api_key: str) -> None:+ """Validate a ``Bearer <token>`` header in constant time.++ Raises ``PermissionError`` on any failure so callers can translate it into+ the framework's 401 (FastAPI workers map this to HTTPException(401)).+ """+ # Fail closed on an empty configured key: with api_key == "" a bare+ # "Bearer " header would otherwise strip to "" and compare equal.+ if not api_key:+ raise PermissionError("worker api key not configured")+ if not authorization or not authorization.startswith("Bearer "):+ raise PermissionError("missing bearer token")+ token = authorization[len("Bearer "):].strip()+ if not token:+ raise PermissionError("missing bearer token")+ # Compare bytes: hmac.compare_digest is timing-safe, and on str operands it+ # raises TypeError for non-ASCII input, which would surface as a 500.+ if not hmac.compare_digest(token.encode("utf-8"), api_key.encode("utf-8")):+ raise PermissionError("invalid api key")+++# ── SSRF-hardened fetch ─────────────────────────────────────────────────────────++# Default cap on a fetched response. Avatar selfies / driving audio / raw meshes+# are comfortably under this; anything larger is almost certainly abuse.+DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB+_MAX_REDIRECTS = 4++# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)++# Hosts that enforce a User-Agent policy refuse httpx's default+# "python-httpx/<version>": upload.wikimedia.org answers it with 403 Forbidden+# and serves the same image to a descriptive agent (measured 2026-10-09, when+# model-triposg failed every job whose reference photo was a Wikimedia link).+# Callers may still pass their own User-Agent; it replaces this one.+DEFAULT_USER_AGENT = "three.ws-worker/1.0 (+https://three.ws)"+++def _fetch_headers(headers: Optional[dict]) -> dict:+ """Caller headers over the default User-Agent, matched case-insensitively."""+ merged = dict(headers or {})+ if not any(str(k).lower() == "user-agent" for k in merged):+ merged["User-Agent"] = DEFAULT_USER_AGENT+ return merged+++class UnsafeUrlError(ValueError):+ """Raised when a URL targets a disallowed scheme or a private/internal IP."""+++def _is_blocked_ip(ip: ipaddress._BaseAddress) -> bool:+ """Reject any address that could reach internal infrastructure.++ Covers loopback, RFC1918 private ranges, link-local (incl. the cloud+ metadata server at 169.254.169.254), unique-local IPv6 (fc00::/7),+ multicast, reserved, unspecified, and any other non-global address (CGNAT+ 100.64.0.0/10 included). IPv4-mapped IPv6 addresses+ are unwrapped first so a mapped private v4 can't slip through.+ """+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:+ ip = ip.ipv4_mapped+ return (+ ip.is_private+ or ip.is_loopback+ or ip.is_link_local+ or ip.is_multicast+ or ip.is_reserved+ or ip.is_unspecified+ # Anything not globally routable, which also covers the CGNAT shared+ # space 100.64.0.0/10 that is_private does not flag.+ or not ip.is_global+ )+++def _resolve_safe_host(host: str) -> None:+ """DNS-resolve ``host`` and raise UnsafeUrlError if ANY address is internal.++ Resolving and checking every returned address closes the gap where a name+ resolves to both a public and a private address.+ """+ if not host:+ raise UnsafeUrlError("missing host")+ try:+ infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)+ except socket.gaierror as exc:+ raise UnsafeUrlError(f"could not resolve host: {host}") from exc++ addrs = {info[4][0] for info in infos}+ if not addrs:+ raise UnsafeUrlError(f"no addresses for host: {host}")+ for addr in addrs:+ ip = ipaddress.ip_address(addr)+ if _is_blocked_ip(ip):+ raise UnsafeUrlError(f"host resolves to a disallowed address: {host}")+++def assert_safe_url(url: str, *, allow_http: bool = False) -> str:+ """Validate scheme + resolved IPs for a single URL. Returns it unchanged.++ https is required by default; ``allow_http`` exists only for callers that+ legitimately need cleartext (not used by the workers).+ """+ parts = urlsplit(url)+ scheme = parts.scheme.lower()+ allowed = ("https", "http") if allow_http else ("https",)+ if scheme not in allowed:+ raise UnsafeUrlError(f"scheme not allowed: {scheme or '(none)'}")+ if not parts.hostname:+ raise UnsafeUrlError("missing host")+ _resolve_safe_host(parts.hostname)+ return url+++def fetch_remote_bytes(+ url: str,+ *,+ timeout: float = 30.0,+ max_bytes: int = DEFAULT_MAX_BYTES,+ allow_http: bool = False,+ headers: Optional[dict] = None,+) -> bytes:+ """Fetch ``url`` safely and return its body bytes.++ Hardening applied:+ * https-only (unless ``allow_http``).+ * every URL - including each redirect target - is DNS-resolved and+ rejected if it maps to a private/loopback/link-local/metadata address.+ * redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is+ re-validated; httpx's own redirect following is disabled.+ * the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.++ Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /+ ``ValueError`` for transport or size failures.+ """+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:+ current = assert_safe_url(url, allow_http=allow_http)+ with httpx.Client(follow_redirects=False, timeout=timeout) as client:+ for _ in range(_MAX_REDIRECTS + 1):+ with client.stream("GET", current, headers=_fetch_headers(headers)) as resp:+ if resp.is_redirect:+ location = resp.headers.get("location")+ if not location:+ raise UnsafeUrlError("redirect without Location header")+ current = assert_safe_url(+ str(resp.url.join(location)), allow_http=allow_http+ )+ continue+ resp.raise_for_status()+ declared = resp.headers.get("content-length")+ if declared is not None and declared.isdigit() and int(declared) > max_bytes:+ raise ValueError(f"response too large: {declared} bytes")+ chunks: list[bytes] = []+ total = 0+ for chunk in resp.iter_bytes(65536):+ total += len(chunk)+ if total > max_bytes:+ raise ValueError(f"response exceeded {max_bytes} bytes")+ chunks.append(chunk)+ return b"".join(chunks)+ raise UnsafeUrlError(f"too many redirects (>{_MAX_REDIRECTS})")+++async def fetch_remote_bytes_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int = DEFAULT_MAX_BYTES,+ allow_http: bool = False,+ headers: Optional[dict] = None,+) -> bytes:+ """Async variant of ``fetch_remote_bytes`` using a caller-provided client.++ The provided ``client`` MUST be constructed with ``follow_redirects=False``+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant.+ """+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:+ current = assert_safe_url(url, allow_http=allow_http)+ for _ in range(_MAX_REDIRECTS + 1):+ async with client.stream("GET", current, headers=_fetch_headers(headers)) as resp:+ if resp.is_redirect:+ location = resp.headers.get("location")+ if not location:+ raise UnsafeUrlError("redirect without Location header")+ current = assert_safe_url(+ str(resp.url.join(location)), allow_http=allow_http+ )+ continue+ resp.raise_for_status()+ declared = resp.headers.get("content-length")+ if declared is not None and declared.isdigit() and int(declared) > max_bytes:+ raise ValueError(f"response too large: {declared} bytes")+ chunks: list[bytes] = []+ total = 0+ async for chunk in resp.aiter_bytes(65536):+ total += len(chunk)+ if total > max_bytes:+ raise ValueError(f"response exceeded {max_bytes} bytes")+ chunks.append(chunk)+ return b"".join(chunks)+ raise UnsafeUrlError(f"too many redirects (>{_MAX_REDIRECTS})")+++# ── opaque error handling ───────────────────────────────────────────────────────+++def safe_error(exc: BaseException, *, context: str = "") -> str:+ """Log the full traceback server-side and return an opaque client message.++ Never leak ``traceback.format_exc()`` (absolute paths, library versions,+ internal hostnames, fragments of upstream responses) to callers. A short+ correlation id is embedded in both the log line and the returned string so+ operators can join a client-reported failure to the server log.+ """+ correlation_id = uuid.uuid4().hex[:12]+ label = f"{context} " if context else ""+ log.exception("%serror [correlation_id=%s]", label, correlation_id)+ return f"internal error (ref {correlation_id})"diff --git a/workers/rig-nonhuman/Dockerfile b/workers/rig-nonhuman/Dockerfileindex ffad0388a..7a06aa05b 100644--- a/workers/rig-nonhuman/Dockerfile+++ b/workers/rig-nonhuman/Dockerfile@@ -2,6 +2,21 @@# VAST-AI-Research, both MIT, LICENSE files read 2026-10-09. Neither ships a# pip-installable package; both are subprocess CLIs (see pipeline.py), so# this image vendors both checkouts at a pinned commit.+#+# gltfpack (zeux/meshoptimizer, MIT) is pinned by release tag and checksum,+# the same pin workers/segment ships: gltf_meshopt.py shells out to it to+# decode EXT_meshopt_compression before trimesh reads a caller's mesh.+FROM python:3.11-slim AS gltfpack+ARG GLTFPACK_VERSION=v1.2+ARG GLTFPACK_SHA256=ebc236f5f6c08c7e5c5750476a187d24805d44d8c680449c4b7369c333f817b1+RUN apt-get update && apt-get install -y --no-install-recommends curl unzip ca-certificates \+ && curl -fsSL -o /tmp/gltfpack.zip \+ https://github.com/zeux/meshoptimizer/releases/download/${GLTFPACK_VERSION}/gltfpack-ubuntu.zip \+ && echo "${GLTFPACK_SHA256} /tmp/gltfpack.zip" | sha256sum -c - \+ && unzip -j /tmp/gltfpack.zip gltfpack -d /usr/local/bin \+ && chmod +x /usr/local/bin/gltfpack \+ && /usr/local/bin/gltfpack -v+FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04ENV PYTHONDONTWRITEBYTECODE=1 \@@ -20,6 +35,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \&& ln -sf /usr/bin/python3.11 /usr/bin/python \&& python3 -m pip install --upgrade pip setuptools wheel+COPY --from=gltfpack /usr/local/bin/gltfpack /usr/local/bin/gltfpack+# torch first, as its own cached layer, matching SkinTokens' documented# install (the newer and more exacting of the two dependency sets; UniRig# only requires torch>=2.3.1, so this satisfies both).