Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(trellis): bake dinov2 into the image and probe the worker so a latched load stops reading as healthy
nirholas3de4f87dcc7f
diff --git a/api/_lib/forge-health.js b/api/_lib/forge-health.jsindex 7ec4c6eec..ecafbe557 100644--- a/api/_lib/forge-health.js+++ b/api/_lib/forge-health.js@@ -214,7 +214,7 @@ function gcpWorkerProbe(id, urlEnv) {const url = readEnv(urlEnv);const key = readEnv('GCP_RECONSTRUCTION_KEY');if (!url || !key) {- return result(id, 'unconfigured', `The ${label} self-host worker is not deployed on this deployment.`);+ return result(id, 'unconfigured', `The ${label} worker is not configured on this deployment.`);}const started = Date.now();const res = await probeFetch(`${url.replace(/\/+$/, '')}/health`, {@@ -258,6 +258,16 @@ const PROBES = {trellis: probeReplicate,hunyuan3d: gcpWorkerProbe('hunyuan3d', 'GCP_HUNYUAN3D_URL'),triposg: gcpWorkerProbe('triposg', 'GCP_TRIPOSG_URL'),+ // The self-hosted TRELLIS worker is the DEFAULT image backend for the draft and+ // standard tiers, and it was the one platform lane with no live probe: it fell+ // through to the env-presence branch below and reported a flat "ok" whenever+ // MODEL_TRELLIS_URL happened to be set. That is exactly the blind spot this+ // report exists to close. On 2026-09-02 the worker latched a failed model load+ // and answered every job with "pipeline unavailable" for 12 hours while this+ // endpoint still called it healthy, so 70 of 219 generations went terminal with+ // nothing flagging it. It publishes the same /health contract the other GCP+ // workers do, load_error included, so the shared probe reads it as down.+ trellis_selfhost: gcpWorkerProbe('trellis_selfhost', 'MODEL_TRELLIS_URL'),};// The distributed rate-limiter store gates every paid lane: when Redis isdiff --git a/workers/model-trellis/Dockerfile b/workers/model-trellis/Dockerfileindex ffefd3703..138183e50 100644--- a/workers/model-trellis/Dockerfile+++ b/workers/model-trellis/Dockerfile@@ -91,6 +91,28 @@ RUN git clone --recurse-submodules https://github.com/microsoft/TRELLIS.git /app&& git -C /app/TRELLIS checkout 442aa1e1afb9014e80681d3bf604e8d728a86ee7 \&& git -C /app/TRELLIS submodule update --init --recursive+# ── 7. dinov2 image conditioner, baked so the model load never calls out ──────+# pipeline.json sets image_cond_model=dinov2_vitl14_reg, and TRELLIS resolves it+# with torch.hub.load("facebookresearch/dinov2", ...) with no branch pinned. That+# call hits github.com on every load just to discover the default branch, and+# torch re-raises any non-404 there before it ever looks at its own cache. Cloud+# Run egresses from shared Google IPs that GitHub rate-limits, so the load had a+# standing dependency on somebody else's quota: on 2026-09-02 a single "HTTP+# Error 403: rate limit exceeded" failed it, and the latched error held the+# default free image lane down for 12 hours. Baking the repo and its checkpoint+# removes that dependency; main.py routes torch.hub at this checkout.+# The checkpoint filename is the one torch.hub.load_state_dict_from_url derives+# from the upstream URL, so the pre-baked copy is a cache hit rather than a+# second download.+ENV TORCH_HOME=/opt/torch \+ DINOV2_LOCAL_DIR=/opt/dinov2+RUN git clone https://github.com/facebookresearch/dinov2.git /opt/dinov2 \+ && git -C /opt/dinov2 checkout 7764ea0f912e53c92e82eb78a2a1631e92725fc8 \+ && mkdir -p /opt/torch/hub/checkpoints \+ && curl -fsSL -o /opt/torch/hub/checkpoints/dinov2_vitl14_reg4_pretrain.pth \+ https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pth \+ && python3 -c "import os; p='/opt/torch/hub/checkpoints/dinov2_vitl14_reg4_pretrain.pth'; s=os.path.getsize(p); assert s > 1_000_000_000, f'dinov2 checkpoint truncated: {s} bytes'"+WORKDIR /appCOPY worker_security.py /app/worker_security.pyCOPY request_policy.py /app/request_policy.pydiff --git a/workers/model-trellis/main.py b/workers/model-trellis/main.pyindex 50bc0fe2a..ba583fdbf 100644--- a/workers/model-trellis/main.py+++ b/workers/model-trellis/main.py@@ -68,7 +68,7 @@ from typing import Optionalfrom urllib.parse import urlsplitimport torch-from fastapi import FastAPI, HTTPException, Header, BackgroundTasks+from fastapi import FastAPI, HTTPException, Header, BackgroundTasks, Responsefrom google.api_core.exceptions import NotFoundfrom google.cloud import storagefrom PIL import Image@@ -122,12 +122,26 @@ WEIGHTS_LOCAL_DIR = os.environ.get("WEIGHTS_LOCAL_DIR", "/tmp/trellis-weights")REMBG_SERVICE_URL = os.environ.get("REMBG_SERVICE_URL", "").rstrip("/")REMBG_DEFAULT_MODEL = os.environ.get("REMBG_MODEL", "isnet-general-use")REMBG_TIMEOUT_S = float(os.environ.get("REMBG_TIMEOUT_S", "90"))+# The model load is retried before it is treated as terminal. Every failure that+# has actually hit this path was transient upstream state (a rate-limited weight+# fetch, a stalled GCS read), not a code fault, so a single exception must not+# decide the fate of the instance. See _load_pipeline_bg for the incident this+# encodes.+MODEL_LOAD_ATTEMPTS = int(os.environ.get("MODEL_LOAD_ATTEMPTS", "4"))+MODEL_LOAD_RETRY_BASE_S = float(os.environ.get("MODEL_LOAD_RETRY_BASE_S", "15"))+MODEL_LOAD_RETRY_CAP_S = float(os.environ.get("MODEL_LOAD_RETRY_CAP_S", "120"))+# TRELLIS's image conditioner (pipeline.json: image_cond_model=dinov2_vitl14_reg)+# is baked into the image here so loading it never reaches the public internet.+DINOV2_LOCAL_DIR = os.environ.get("DINOV2_LOCAL_DIR", "/opt/dinov2")+DINOV2_HUB_REPO = "facebookresearch/dinov2"_pipeline = None_bucket: Optional[storage.Bucket] = None_sem: Optional[asyncio.Semaphore] = None_ready: Optional[asyncio.Event] = None_load_error: Optional[str] = None+_load_attempts = 0+_dinov2_pinned = False# In-memory cache only — Cloud Run runs this service across up to _MAX_INSTANCES# containers with no session affinity, so a POST /infer and a later GET# /tasks/:id can land on different instances. The durable source of truth is@@ -179,6 +193,46 @@ def _stage_weights_local() -> Optional[str]:return None+def _pin_dinov2_to_local_checkout() -> bool:+ """Resolve TRELLIS's image conditioner from the image-baked dinov2 checkout.++ TRELLIS builds that conditioner with torch.hub.load("facebookresearch/dinov2",+ ...), passing no branch. torch.hub therefore calls github.com on EVERY load+ just to discover the repo's default branch, and it re-raises any non-404 HTTP+ error out of that probe before it ever consults its own cache. Cloud Run+ egresses from shared Google IPs that GitHub rate-limits hard, so that probe+ is a standing outage risk: one "HTTP Error 403: rate limit exceeded" on+ 2026-09-02 failed the load, and the latched error then took the default free+ image lane down for 12 hours. The repo and its checkpoint are baked into the+ image (see the Dockerfile), so point torch.hub at the local checkout and drop+ the network dependency entirely.++ Returns False when the checkout is absent, which leaves the historical GitHub+ path exactly as it was: this can only remove a failure mode, never add one.+ """+ global _dinov2_pinned+ if _dinov2_pinned:+ return True+ if not os.path.isfile(os.path.join(DINOV2_LOCAL_DIR, "hubconf.py")):+ log.warning(+ "dinov2 checkout missing at %s; torch.hub will resolve it over the network",+ DINOV2_LOCAL_DIR,+ )+ return False+ original_load = torch.hub.load++ def _load_local_first(repo_or_dir, model, *args, **kwargs):+ if isinstance(repo_or_dir, str) and repo_or_dir.startswith(DINOV2_HUB_REPO):+ kwargs["source"] = "local"+ return original_load(DINOV2_LOCAL_DIR, model, *args, **kwargs)+ return original_load(repo_or_dir, model, *args, **kwargs)++ torch.hub.load = _load_local_first+ _dinov2_pinned = True+ log.info("dinov2 pinned to the local checkout at %s", DINOV2_LOCAL_DIR)+ return True++def _load_pipeline():global _pipelinefrom trellis.pipelines import TrellisImageTo3DPipeline@@ -195,18 +249,40 @@ async def _load_pipeline_bg():"""Load the pipeline off the request path and signal readiness when done.Runs the blocking, GPU-bound load in a worker thread so the event loop (and- the HTTP port) stay live. On failure the error is recorded and surfaced via- /health and per-task, rather than crash-looping the container.+ the HTTP port) stay live.++ The load is RETRIED with exponential backoff, and _load_error stays unset+ while attempts remain, so a job that arrives mid-retry waits on _ready+ instead of failing against a half-written verdict. Latching the very first+ exception is what turned a single transient 403 into a 12-hour outage on+ 2026-09-02: the error was cached in memory, every later task failed against+ it instantly, and minScale=1 kept that dead instance resident and in+ rotation. Only once the whole budget is spent does the error latch, and from+ there /health answers 503 and /infer refuses new work, so the instance is+ reported down and the caller fails over instead of queueing behind a corpse."""- global _load_error+ global _load_error, _load_attemptsloop = asyncio.get_event_loop()- try:- await loop.run_in_executor(None, _load_pipeline)- _ready.set()- log.info("TRELLIS pipeline ready")- except Exception as exc: # noqa: BLE001 — surfaced via /health + task status- _load_error = safe_error(exc, context="model load")- log.error("TRELLIS pipeline load FAILED: %s", exc)+ attempts = max(1, MODEL_LOAD_ATTEMPTS)+ for attempt in range(1, attempts + 1):+ _load_attempts = attempt+ try:+ await loop.run_in_executor(None, _load_pipeline)+ _load_error = None+ _ready.set()+ log.info("TRELLIS pipeline ready (attempt %d)", attempt)+ return+ except Exception as exc: # noqa: BLE001 - surfaced via /health + task status+ if attempt >= attempts:+ _load_error = safe_error(exc, context="model load")+ log.error("TRELLIS pipeline load FAILED after %d attempt(s): %s", attempt, exc)+ return+ delay = min(MODEL_LOAD_RETRY_BASE_S * (2 ** (attempt - 1)), MODEL_LOAD_RETRY_CAP_S)+ log.warning(+ "TRELLIS pipeline load attempt %d/%d failed (%s); retrying in %.0fs",+ attempt, attempts, exc, delay,+ )+ await asyncio.sleep(delay)@asynccontextmanager@@ -221,6 +297,7 @@ async def lifespan(app: FastAPI):# load + GPU transfer runs minutes on a cold instance) and the revision would# be marked failed. Backgrounding lets the port open at once; requests that# arrive before the load completes wait on _ready (see _run_inference).+ _pin_dinov2_to_local_checkout()asyncio.create_task(_load_pipeline_bg())log.info("Service starting — pipeline loading in background (max_concurrent=%d)", MAX_CONCURRENT)yield@@ -609,6 +686,18 @@ async def infer(authorization: str = Header(...),) -> dict:_require_api_key(authorization)+ # A latched load failure means this instance can never serve this job, so+ # refuse it at submit time. 503 is what the caller's lane failover reads+ # (api/forge.js isUpstreamUnavailable -> markLaneUnhealthy), which routes the+ # request to another backend. Accepting the job and only failing it minutes+ # later during polling bypasses that failover entirely: it is how 70+ # generations went terminal in the 24 h to 2026-09-03 while the catalog and+ # the health report both still called this lane healthy.+ if _load_error:+ raise HTTPException(+ status_code=503,+ detail=f"pipeline unavailable: {_load_error}",+ )task_id = str(uuid.uuid4())# Persist the "queued" record before responding — a poll can reach a# different instance than this one the moment the 202 lands, and that@@ -656,15 +745,29 @@ async def root() -> dict:@app.get("/health")-async def health() -> dict:+async def health(response: Response) -> dict:+ """Readiness, answered honestly.++ This used to return ok:true unconditionally, including while the pipeline+ was permanently dead, so nothing upstream could tell a working instance from+ a poisoned one. A spent load budget now answers 503: the platform health+ probe reads that as down (api/_lib/forge-health.js), and a Cloud Run liveness+ probe pointed here recycles the container instead of leaving it resident.+ A load still in progress stays 200 so an ordinary cold start is never killed+ mid-load.+ """+ dead = _load_error is not None+ if dead:+ response.status_code = 503return {- "ok": True,+ "ok": not dead,"model": "trellis-image-large","gpu_available": torch.cuda.is_available(),"gpu_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,"pipeline_loaded": _pipeline is not None,"ready": bool(_ready and _ready.is_set()),"load_error": _load_error,+ "load_attempts": _load_attempts,"tiers": list(TIER_PRESETS),"default_quality": QUALITY_DEFAULTS,"rembg_matte": bool(REMBG_SERVICE_URL),