Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(trellis): bake dinov2 into the image and probe the worker so a latched load stops reading as healthy

nirholas3de4f87dcc7f
diff --git a/api/_lib/forge-health.js b/api/_lib/forge-health.js
index 7ec4c6eec..ecafbe557 100644
--- a/api/_lib/forge-health.js
+++ b/api/_lib/forge-health.js
@@ -214,7 +214,7 @@ function gcpWorkerProbe(id, urlEnv) {
const url = readEnv(urlEnv);
const key = readEnv('GCP_RECONSTRUCTION_KEY');
if (!url || !key) {
- return result(id, 'unconfigured', `The ${label} self-host worker is not deployed on this deployment.`);
+ return result(id, 'unconfigured', `The ${label} worker is not configured on this deployment.`);
}
const started = Date.now();
const res = await probeFetch(`${url.replace(/\/+$/, '')}/health`, {
@@ -258,6 +258,16 @@ const PROBES = {
trellis: probeReplicate,
hunyuan3d: gcpWorkerProbe('hunyuan3d', 'GCP_HUNYUAN3D_URL'),
triposg: gcpWorkerProbe('triposg', 'GCP_TRIPOSG_URL'),
+ // The self-hosted TRELLIS worker is the DEFAULT image backend for the draft and
+ // standard tiers, and it was the one platform lane with no live probe: it fell
+ // through to the env-presence branch below and reported a flat "ok" whenever
+ // MODEL_TRELLIS_URL happened to be set. That is exactly the blind spot this
+ // report exists to close. On 2026-09-02 the worker latched a failed model load
+ // and answered every job with "pipeline unavailable" for 12 hours while this
+ // endpoint still called it healthy, so 70 of 219 generations went terminal with
+ // nothing flagging it. It publishes the same /health contract the other GCP
+ // workers do, load_error included, so the shared probe reads it as down.
+ trellis_selfhost: gcpWorkerProbe('trellis_selfhost', 'MODEL_TRELLIS_URL'),
};
// The distributed rate-limiter store gates every paid lane: when Redis is
diff --git a/workers/model-trellis/Dockerfile b/workers/model-trellis/Dockerfile
index ffefd3703..138183e50 100644
--- a/workers/model-trellis/Dockerfile
+++ b/workers/model-trellis/Dockerfile
@@ -91,6 +91,28 @@ RUN git clone --recurse-submodules https://github.com/microsoft/TRELLIS.git /app
&& git -C /app/TRELLIS checkout 442aa1e1afb9014e80681d3bf604e8d728a86ee7 \
&& git -C /app/TRELLIS submodule update --init --recursive
+# ── 7. dinov2 image conditioner, baked so the model load never calls out ──────
+# pipeline.json sets image_cond_model=dinov2_vitl14_reg, and TRELLIS resolves it
+# with torch.hub.load("facebookresearch/dinov2", ...) with no branch pinned. That
+# call hits github.com on every load just to discover the default branch, and
+# torch re-raises any non-404 there before it ever looks at its own cache. Cloud
+# Run egresses from shared Google IPs that GitHub rate-limits, so the load had a
+# standing dependency on somebody else's quota: on 2026-09-02 a single "HTTP
+# Error 403: rate limit exceeded" failed it, and the latched error held the
+# default free image lane down for 12 hours. Baking the repo and its checkpoint
+# removes that dependency; main.py routes torch.hub at this checkout.
+# The checkpoint filename is the one torch.hub.load_state_dict_from_url derives
+# from the upstream URL, so the pre-baked copy is a cache hit rather than a
+# second download.
+ENV TORCH_HOME=/opt/torch \
+ DINOV2_LOCAL_DIR=/opt/dinov2
+RUN git clone https://github.com/facebookresearch/dinov2.git /opt/dinov2 \
+ && git -C /opt/dinov2 checkout 7764ea0f912e53c92e82eb78a2a1631e92725fc8 \
+ && mkdir -p /opt/torch/hub/checkpoints \
+ && curl -fsSL -o /opt/torch/hub/checkpoints/dinov2_vitl14_reg4_pretrain.pth \
+ https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pth \
+ && python3 -c "import os; p='/opt/torch/hub/checkpoints/dinov2_vitl14_reg4_pretrain.pth'; s=os.path.getsize(p); assert s > 1_000_000_000, f'dinov2 checkpoint truncated: {s} bytes'"
+
WORKDIR /app
COPY worker_security.py /app/worker_security.py
COPY request_policy.py /app/request_policy.py
diff --git a/workers/model-trellis/main.py b/workers/model-trellis/main.py
index 50bc0fe2a..ba583fdbf 100644
--- a/workers/model-trellis/main.py
+++ b/workers/model-trellis/main.py
@@ -68,7 +68,7 @@ from typing import Optional
from urllib.parse import urlsplit
import torch
-from fastapi import FastAPI, HTTPException, Header, BackgroundTasks
+from fastapi import FastAPI, HTTPException, Header, BackgroundTasks, Response
from google.api_core.exceptions import NotFound
from google.cloud import storage
from PIL import Image
@@ -122,12 +122,26 @@ WEIGHTS_LOCAL_DIR = os.environ.get("WEIGHTS_LOCAL_DIR", "/tmp/trellis-weights")
REMBG_SERVICE_URL = os.environ.get("REMBG_SERVICE_URL", "").rstrip("/")
REMBG_DEFAULT_MODEL = os.environ.get("REMBG_MODEL", "isnet-general-use")
REMBG_TIMEOUT_S = float(os.environ.get("REMBG_TIMEOUT_S", "90"))
+# The model load is retried before it is treated as terminal. Every failure that
+# has actually hit this path was transient upstream state (a rate-limited weight
+# fetch, a stalled GCS read), not a code fault, so a single exception must not
+# decide the fate of the instance. See _load_pipeline_bg for the incident this
+# encodes.
+MODEL_LOAD_ATTEMPTS = int(os.environ.get("MODEL_LOAD_ATTEMPTS", "4"))
+MODEL_LOAD_RETRY_BASE_S = float(os.environ.get("MODEL_LOAD_RETRY_BASE_S", "15"))
+MODEL_LOAD_RETRY_CAP_S = float(os.environ.get("MODEL_LOAD_RETRY_CAP_S", "120"))
+# TRELLIS's image conditioner (pipeline.json: image_cond_model=dinov2_vitl14_reg)
+# is baked into the image here so loading it never reaches the public internet.
+DINOV2_LOCAL_DIR = os.environ.get("DINOV2_LOCAL_DIR", "/opt/dinov2")
+DINOV2_HUB_REPO = "facebookresearch/dinov2"
_pipeline = None
_bucket: Optional[storage.Bucket] = None
_sem: Optional[asyncio.Semaphore] = None
_ready: Optional[asyncio.Event] = None
_load_error: Optional[str] = None
+_load_attempts = 0
+_dinov2_pinned = False
# In-memory cache only — Cloud Run runs this service across up to _MAX_INSTANCES
# containers with no session affinity, so a POST /infer and a later GET
# /tasks/:id can land on different instances. The durable source of truth is
@@ -179,6 +193,46 @@ def _stage_weights_local() -> Optional[str]:
return None
+def _pin_dinov2_to_local_checkout() -> bool:
+ """Resolve TRELLIS's image conditioner from the image-baked dinov2 checkout.
+
+ TRELLIS builds that conditioner with torch.hub.load("facebookresearch/dinov2",
+ ...), passing no branch. torch.hub therefore calls github.com on EVERY load
+ just to discover the repo's default branch, and it re-raises any non-404 HTTP
+ error out of that probe before it ever consults its own cache. Cloud Run
+ egresses from shared Google IPs that GitHub rate-limits hard, so that probe
+ is a standing outage risk: one "HTTP Error 403: rate limit exceeded" on
+ 2026-09-02 failed the load, and the latched error then took the default free
+ image lane down for 12 hours. The repo and its checkpoint are baked into the
+ image (see the Dockerfile), so point torch.hub at the local checkout and drop
+ the network dependency entirely.
+
+ Returns False when the checkout is absent, which leaves the historical GitHub
+ path exactly as it was: this can only remove a failure mode, never add one.
+ """
+ global _dinov2_pinned
+ if _dinov2_pinned:
+ return True
+ if not os.path.isfile(os.path.join(DINOV2_LOCAL_DIR, "hubconf.py")):
+ log.warning(
+ "dinov2 checkout missing at %s; torch.hub will resolve it over the network",
+ DINOV2_LOCAL_DIR,
+ )
+ return False
+ original_load = torch.hub.load
+
+ def _load_local_first(repo_or_dir, model, *args, **kwargs):
+ if isinstance(repo_or_dir, str) and repo_or_dir.startswith(DINOV2_HUB_REPO):
+ kwargs["source"] = "local"
+ return original_load(DINOV2_LOCAL_DIR, model, *args, **kwargs)
+ return original_load(repo_or_dir, model, *args, **kwargs)
+
+ torch.hub.load = _load_local_first
+ _dinov2_pinned = True
+ log.info("dinov2 pinned to the local checkout at %s", DINOV2_LOCAL_DIR)
+ return True
+
+
def _load_pipeline():
global _pipeline
from trellis.pipelines import TrellisImageTo3DPipeline
@@ -195,18 +249,40 @@ async def _load_pipeline_bg():
"""Load the pipeline off the request path and signal readiness when done.
Runs the blocking, GPU-bound load in a worker thread so the event loop (and
- the HTTP port) stay live. On failure the error is recorded and surfaced via
- /health and per-task, rather than crash-looping the container.
+ the HTTP port) stay live.
+
+ The load is RETRIED with exponential backoff, and _load_error stays unset
+ while attempts remain, so a job that arrives mid-retry waits on _ready
+ instead of failing against a half-written verdict. Latching the very first
+ exception is what turned a single transient 403 into a 12-hour outage on
+ 2026-09-02: the error was cached in memory, every later task failed against
+ it instantly, and minScale=1 kept that dead instance resident and in
+ rotation. Only once the whole budget is spent does the error latch, and from
+ there /health answers 503 and /infer refuses new work, so the instance is
+ reported down and the caller fails over instead of queueing behind a corpse.
"""
- global _load_error
+ global _load_error, _load_attempts
loop = asyncio.get_event_loop()
- try:
- await loop.run_in_executor(None, _load_pipeline)
- _ready.set()
- log.info("TRELLIS pipeline ready")
- except Exception as exc: # noqa: BLE001 — surfaced via /health + task status
- _load_error = safe_error(exc, context="model load")
- log.error("TRELLIS pipeline load FAILED: %s", exc)
+ attempts = max(1, MODEL_LOAD_ATTEMPTS)
+ for attempt in range(1, attempts + 1):
+ _load_attempts = attempt
+ try:
+ await loop.run_in_executor(None, _load_pipeline)
+ _load_error = None
+ _ready.set()
+ log.info("TRELLIS pipeline ready (attempt %d)", attempt)
+ return
+ except Exception as exc: # noqa: BLE001 - surfaced via /health + task status
+ if attempt >= attempts:
+ _load_error = safe_error(exc, context="model load")
+ log.error("TRELLIS pipeline load FAILED after %d attempt(s): %s", attempt, exc)
+ return
+ delay = min(MODEL_LOAD_RETRY_BASE_S * (2 ** (attempt - 1)), MODEL_LOAD_RETRY_CAP_S)
+ log.warning(
+ "TRELLIS pipeline load attempt %d/%d failed (%s); retrying in %.0fs",
+ attempt, attempts, exc, delay,
+ )
+ await asyncio.sleep(delay)
@asynccontextmanager
@@ -221,6 +297,7 @@ async def lifespan(app: FastAPI):
# load + GPU transfer runs minutes on a cold instance) and the revision would
# be marked failed. Backgrounding lets the port open at once; requests that
# arrive before the load completes wait on _ready (see _run_inference).
+ _pin_dinov2_to_local_checkout()
asyncio.create_task(_load_pipeline_bg())
log.info("Service starting — pipeline loading in background (max_concurrent=%d)", MAX_CONCURRENT)
yield
@@ -609,6 +686,18 @@ async def infer(
authorization: str = Header(...),
) -> dict:
_require_api_key(authorization)
+ # A latched load failure means this instance can never serve this job, so
+ # refuse it at submit time. 503 is what the caller's lane failover reads
+ # (api/forge.js isUpstreamUnavailable -> markLaneUnhealthy), which routes the
+ # request to another backend. Accepting the job and only failing it minutes
+ # later during polling bypasses that failover entirely: it is how 70
+ # generations went terminal in the 24 h to 2026-09-03 while the catalog and
+ # the health report both still called this lane healthy.
+ if _load_error:
+ raise HTTPException(
+ status_code=503,
+ detail=f"pipeline unavailable: {_load_error}",
+ )
task_id = str(uuid.uuid4())
# Persist the "queued" record before responding — a poll can reach a
# different instance than this one the moment the 202 lands, and that
@@ -656,15 +745,29 @@ async def root() -> dict:
@app.get("/health")
-async def health() -> dict:
+async def health(response: Response) -> dict:
+ """Readiness, answered honestly.
+
+ This used to return ok:true unconditionally, including while the pipeline
+ was permanently dead, so nothing upstream could tell a working instance from
+ a poisoned one. A spent load budget now answers 503: the platform health
+ probe reads that as down (api/_lib/forge-health.js), and a Cloud Run liveness
+ probe pointed here recycles the container instead of leaving it resident.
+ A load still in progress stays 200 so an ordinary cold start is never killed
+ mid-load.
+ """
+ dead = _load_error is not None
+ if dead:
+ response.status_code = 503
return {
- "ok": True,
+ "ok": not dead,
"model": "trellis-image-large",
"gpu_available": torch.cuda.is_available(),
"gpu_name": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,
"pipeline_loaded": _pipeline is not None,
"ready": bool(_ready and _ready.is_set()),
"load_error": _load_error,
+ "load_attempts": _load_attempts,
"tiers": list(TIER_PRESETS),
"default_quality": QUALITY_DEFAULTS,
"rembg_matte": bool(REMBG_SERVICE_URL),