Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(workers): free GPU memory after every job so a lane stops dying after N generations

nirholas17215b5c26d7
diff --git a/workers/model-hunyuan3d/main.py b/workers/model-hunyuan3d/main.py
index a12e88223..c2663df2c 100644
--- a/workers/model-hunyuan3d/main.py
+++ b/workers/model-hunyuan3d/main.py
@@ -30,6 +30,7 @@ Environment variables:
from __future__ import annotations
import asyncio
+import gc
import io
import json
import logging
@@ -457,6 +458,29 @@ async def _run_inference(task_id: str, images: list[str], body_type: str, qualit
error=safe_error(exc, context=f"[{task_id}] inference"),
elapsed_ms=int((time.time() - t0) * 1000),
)
+ finally:
+ # Reclaim before releasing the semaphore, so the next inference
+ # starts against a drained device. See _release_gpu_memory.
+ await loop.run_in_executor(None, _release_gpu_memory)
+
+
+def _release_gpu_memory() -> None:
+ """Return a finished job's GPU allocations to the driver.
+
+ Blocking, so callers run it in the executor. Safe on CPU-only hosts because
+ every CUDA call is gated on availability.
+
+ Torch's caching allocator keeps freed blocks reserved rather than returning
+ them, so without this an instance's usable VRAM only ever shrinks and a
+ long-lived one eventually fails every job it accepts with a cudaMalloc
+ error while its health check still passes. That is exactly how the sibling
+ TRELLIS lane went from 1,189 generations on 2026-09-06 to a 100% failure
+ rate on 2026-09-09; this lane had the same gap.
+ """
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ torch.cuda.ipc_collect()
class InferRequest(BaseModel):
diff --git a/workers/model-text2motion/main.py b/workers/model-text2motion/main.py
index 2ae81ddb8..0458133d1 100644
--- a/workers/model-text2motion/main.py
+++ b/workers/model-text2motion/main.py
@@ -31,6 +31,7 @@ Environment:
from __future__ import annotations
import asyncio
+import gc
import json
import logging
import os
@@ -217,6 +218,35 @@ async def _run_inference(task_id: str, prompt: str, n_frames: int, fps: int) ->
msg = safe_error(exc, context=f"text2motion task {task_id}")
_tasks[task_id].update({"status": "failed", "error": msg})
log.exception("task %s failed", task_id)
+ finally:
+ # Reclaim before releasing the semaphore, so the next inference
+ # starts against a drained device. See _release_gpu_memory.
+ await loop.run_in_executor(None, _release_gpu_memory)
+
+
+def _release_gpu_memory() -> None:
+ """Return a finished job's GPU allocations to the driver.
+
+ Blocking, so callers run it in the executor. torch is imported lazily and
+ the failure is swallowed because this module is deliberately importable
+ without torch or MDM installed (see _load_model); on such a host there is
+ no device to reclaim and the call is a no-op.
+
+ Torch's caching allocator keeps freed blocks reserved rather than returning
+ them, so without this an instance's usable VRAM only ever shrinks and a
+ long-lived one eventually fails every job it accepts with a cudaMalloc
+ error while its health check still passes. That is exactly how the sibling
+ TRELLIS lane went from 1,189 generations on 2026-09-06 to a 100% failure
+ rate on 2026-09-09; this lane had the same gap.
+ """
+ gc.collect()
+ try:
+ import torch
+ except ImportError:
+ return
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ torch.cuda.ipc_collect()
def _safe_name(prompt: str) -> str:
diff --git a/workers/model-trellis/main.py b/workers/model-trellis/main.py
index b96c012ec..00008547c 100644
--- a/workers/model-trellis/main.py
+++ b/workers/model-trellis/main.py
@@ -58,6 +58,7 @@ from __future__ import annotations
import asyncio
import base64
+import gc
import io
import json
import logging
@@ -664,6 +665,36 @@ async def _run_inference(
error=safe_error(exc, context=f"[{task_id}] inference"),
elapsed_ms=int((time.time() - t0) * 1000),
)
+ finally:
+ # Hand this job's GPU memory back before releasing the semaphore, so
+ # the next inference starts against a drained device and the reclaim
+ # itself never races another job's allocations.
+ #
+ # Nothing else frees it. Torch's caching allocator holds freed blocks
+ # in reserve, and nvdiffrast's rasterizer allocates through its own
+ # cudaMalloc outside that cache, so an instance's usable VRAM only
+ # ever shrinks. The instance stays healthy for a while and then fails
+ # EVERY later job with `Cuda error: 2[cudaMalloc(&m_gpuPtr, bytes);]`
+ # out of postprocess_mesh's _fill_holes, while its health check keeps
+ # passing and Cloud Run keeps routing to it.
+ #
+ # That is exactly how this lane died: 1,189 generations on 2026-09-06,
+ # then on 2026-09-09 thirteen jobs succeeded after a restart and every
+ # single job after them failed. Restarting only buys another thirteen.
+ await loop.run_in_executor(None, _release_gpu_memory)
+
+
+def _release_gpu_memory() -> None:
+ """Return a finished job's GPU allocations to the driver.
+
+ Blocking, so callers run it in the executor. Safe on CPU-only hosts (the
+ unit tests import this module without a GPU) because every CUDA call is
+ gated on availability.
+ """
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ torch.cuda.ipc_collect()
class InferRequest(BaseModel):