Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(trellis): cap malloc arenas, recycle on memory pressure and expire orphaned tasks

nirholas7f0f36e71c5a
diff --git a/workers/model-trellis/Dockerfile b/workers/model-trellis/Dockerfile
index 1eab909c0..d5ba7094a 100644
--- a/workers/model-trellis/Dockerfile
+++ b/workers/model-trellis/Dockerfile
@@ -25,7 +25,12 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
ATTN_BACKEND=xformers \
SPCONV_ALGO=native \
# TRELLIS repo lives here and is imported off PYTHONPATH (not a pip package).
- PYTHONPATH=/app/TRELLIS
+ PYTHONPATH=/app/TRELLIS \
+ # Cap glibc at two malloc arenas. Every job runs on executor threads, and by
+ # default each thread grows its own arena that keeps freed heap forever: the
+ # instance ratcheted from 46% to 91% of its 32 GiB across a day of jobs and
+ # was OOM-killed mid-job (see instance_health.py).
+ MALLOC_ARENA_MAX=2
# libusb-1.0-0 and libegl1: open3d (imported by TRELLIS's pipelines package)
# dlopens both at import time. Without them the model load dies with
@@ -125,17 +130,19 @@ WORKDIR /app
COPY worker_security.py /app/worker_security.py
COPY request_policy.py /app/request_policy.py
COPY storage_backend.py /app/storage_backend.py
+COPY instance_health.py /app/instance_health.py
COPY main.py /app/main.py
-COPY test_request_policy.py test_storage_backend.py test_app_contract.py /app/
+COPY test_request_policy.py test_storage_backend.py test_instance_health.py test_app_contract.py /app/
# Build-time gate, neither half of which needs a GPU:
# 1. The quality tiers, the clamps that keep a caller value from crashing the
-# nvdiffrast bake, and the image-fetch retry policy are pure logic, proven
-# here rather than in production.
+# nvdiffrast bake, the image-fetch retry policy, and the memory-recycle and
+# orphan-expiry decisions are pure logic, proven here rather than in
+# production.
# 2. The served contract, against the exact dependency set that will run: the
# TRELLIS import chain (FlexiCubes submodule included), the auth boundary,
# the probe surfaces, request validation, and the SSRF guard.
-RUN python3 -m pytest /app/test_request_policy.py /app/test_storage_backend.py -q \
+RUN python3 -m pytest /app/test_request_policy.py /app/test_storage_backend.py /app/test_instance_health.py -q \
&& python3 /app/test_app_contract.py
EXPOSE 8080
diff --git a/workers/model-trellis/instance_health.py b/workers/model-trellis/instance_health.py
new file mode 100644
index 000000000..124dbc40a
--- /dev/null
+++ b/workers/model-trellis/instance_health.py
@@ -0,0 +1,219 @@
+"""Instance self-care for the TRELLIS worker: host memory and task liveness.
+
+Pure stdlib, no torch and no CUDA, so every decision here is unit tested on any
+machine (see test_instance_health.py) and main.py only wires it in.
+
+Why this module exists (production, 2026-10-03 to 2026-10-08):
+
+ * The single warm L4 instance was OOM-killed twelve times in five days
+ ("Container terminated on signal 9"). Cloud Run's memory utilization for
+ the revision climbed in steps after each heavy job (0.46 -> 0.56 -> 0.71 ->
+ 0.81 -> 0.88 -> 0.91) and never came back down, then a mesh postprocess
+ (decimate, invisible-face removal, UV unwrap, texture bake) pushed it past
+ the 32 GiB ceiling, which is the most memory Cloud Run gives an L4 instance.
+ That is glibc keeping freed heap in per-thread arenas: the pipeline runs in
+ executor threads, each of which grows its own arena, and nothing hands the
+ freed pages back to the kernel.
+ * Every kill took the in-flight job with it, and the queued job behind it.
+ Their durable GCS records stayed "running" until a 30 minute orphan window
+ expired, long after the caller's poll budget, so the forge's lane failover
+ never saw a failure to act on and the user just watched a spinner.
+
+The fixes are split by what they guard against:
+
+ trim_heap() returns freed heap to the kernel after every job.
+ read_memory() reports what the instance actually holds, so each job
+ logs its footprint and the recycle guard has a number.
+ should_recycle() decides when an instance has ratcheted far enough that
+ it should restart BETWEEN jobs rather than die inside one.
+ task_is_orphaned() lets a heartbeat-refreshed record expire in minutes, not
+ half an hour, once its runner is gone.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import ctypes.util
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+# How often a live task refreshes its durable record, and how long a queued or
+# running record may go without a refresh before it is declared orphaned. Six
+# missed beats is far beyond any scheduling hiccup on a healthy instance, and
+# still well inside the forge client's ~300 s poll budget, so the failure lands
+# while a client is still listening and the lane failover can redispatch.
+HEARTBEAT_SECS = 30.0
+ORPHAN_AFTER_SECS = 180.0
+
+# Recycle once the instance holds this share of its memory limit after a job...
+RECYCLE_FRACTION = float(os.environ.get("MEMORY_RECYCLE_FRACTION", "0.80"))
+# ...and only if it has grown at least this much past its post-load baseline. The
+# growth floor is what stops a mismeasured or simply large baseline from turning
+# into a restart after every single job.
+RECYCLE_MIN_GROWTH = float(os.environ.get("MEMORY_RECYCLE_MIN_GROWTH", "0.15"))
+
+_CGROUP_V2 = "/sys/fs/cgroup"
+_CGROUP_V1 = "/sys/fs/cgroup/memory"
+_MEMINFO = "/proc/meminfo"
+
+
+@dataclass(frozen=True)
+class MemorySample:
+ used_bytes: int
+ limit_bytes: int
+ source: str
+
+ @property
+ def fraction(self) -> float:
+ return self.used_bytes / self.limit_bytes if self.limit_bytes else 0.0
+
+ def describe(self) -> str:
+ gib = 1024 ** 3
+ return (
+ f"{self.used_bytes / gib:.1f}/{self.limit_bytes / gib:.1f} GiB "
+ f"({self.fraction:.0%}, {self.source})"
+ )
+
+
+def _read(path: str) -> Optional[str]:
+ try:
+ with open(path, encoding="ascii") as fh:
+ return fh.read()
+ except OSError:
+ return None
+
+
+def _stat_fields(text: str) -> dict[str, int]:
+ fields: dict[str, int] = {}
+ for line in text.splitlines():
+ parts = line.split()
+ if len(parts) >= 2 and parts[1].isdigit():
+ fields[parts[0].rstrip(":")] = int(parts[1])
+ return fields
+
+
+def _from_cgroup_v2(root: str) -> Optional[MemorySample]:
+ stat, limit = _read(os.path.join(root, "memory.stat")), _read(os.path.join(root, "memory.max"))
+ if stat is None or limit is None or not limit.strip().isdigit():
+ return None
+ fields = _stat_fields(stat)
+ if "anon" not in fields:
+ return None
+ # anon is the process heap; shmem is tmpfs, which is where Cloud Run's
+ # in-memory filesystem (and so the staged weights in /tmp) is charged.
+ # Page cache is left out on purpose: the kernel reclaims it before it OOMs.
+ return MemorySample(fields["anon"] + fields.get("shmem", 0), int(limit.strip()), "cgroup2")
+
+
+def _from_cgroup_v1(root: str) -> Optional[MemorySample]:
+ stat = _read(os.path.join(root, "memory.stat"))
+ limit = _read(os.path.join(root, "memory.limit_in_bytes"))
+ if stat is None or limit is None or not limit.strip().isdigit():
+ return None
+ fields = _stat_fields(stat)
+ if "total_rss" not in fields:
+ return None
+ limit_bytes = int(limit.strip())
+ # An unlimited v1 group reports a sentinel near 2**63; it says nothing about
+ # this instance's ceiling, so defer to /proc/meminfo instead.
+ if limit_bytes >= 1 << 60:
+ return None
+ return MemorySample(fields["total_rss"] + fields.get("total_shmem", 0), limit_bytes, "cgroup1")
+
+
+def _from_meminfo(path: str) -> Optional[MemorySample]:
+ text = _read(path)
+ if text is None:
+ return None
+ fields = _stat_fields(text)
+ total, available = fields.get("MemTotal"), fields.get("MemAvailable")
+ if not total or available is None:
+ return None
+ # /proc/meminfo is in KiB. MemAvailable already discounts reclaimable cache.
+ return MemorySample((total - available) * 1024, total * 1024, "meminfo")
+
+
+def read_memory(
+ cgroup_v2_root: str = _CGROUP_V2,
+ cgroup_v1_root: str = _CGROUP_V1,
+ meminfo_path: str = _MEMINFO,
+) -> Optional[MemorySample]:
+ """Best available view of the memory this instance holds against its limit.
+
+ Never raises. Returns None when no source is readable, and every caller
+ treats None as "unknown", which never triggers a recycle.
+ """
+ for reader, arg in (
+ (_from_cgroup_v2, cgroup_v2_root),
+ (_from_cgroup_v1, cgroup_v1_root),
+ (_from_meminfo, meminfo_path),
+ ):
+ try:
+ sample = reader(arg)
+ except (ValueError, ZeroDivisionError):
+ sample = None
+ if sample is not None and sample.limit_bytes > 0:
+ return sample
+ return None
+
+
+def should_recycle(
+ current: Optional[MemorySample],
+ baseline: Optional[MemorySample],
+ threshold: float = RECYCLE_FRACTION,
+ min_growth: float = RECYCLE_MIN_GROWTH,
+) -> bool:
+ """True when the instance should restart between jobs.
+
+ Both conditions must hold: the instance is near its ceiling, AND it got
+ there by growing since the model finished loading. Without a baseline (the
+ load has not completed, or memory is unreadable) the answer is always no.
+ """
+ if current is None or baseline is None:
+ return False
+ return current.fraction >= threshold and current.fraction - baseline.fraction >= min_growth
+
+
+_libc = None
+
+
+def trim_heap() -> bool:
+ """Hand freed heap pages back to the kernel (glibc malloc_trim).
+
+ Returns False on a libc without malloc_trim (musl, macOS), where there is
+ nothing to do. Never raises.
+ """
+ global _libc
+ try:
+ if _libc is None:
+ _libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")
+ trim = getattr(_libc, "malloc_trim", None)
+ if trim is None:
+ return False
+ trim.argtypes = [ctypes.c_size_t]
+ trim.restype = ctypes.c_int
+ trim(0)
+ return True
+ except OSError:
+ return False
+
+
+def task_is_orphaned(
+ status: Optional[str],
+ updated_at: object,
+ now: float,
+ orphan_after: float = ORPHAN_AFTER_SECS,
+) -> bool:
+ """True when a queued or running record has gone stale.
+
+ A live runner refreshes updated_at every HEARTBEAT_SECS, so a record that
+ has not moved for orphan_after seconds belongs to an instance that died.
+ A record with no usable timestamp cannot prove it is alive, so it counts as
+ orphaned too. Terminal and unknown statuses never are.
+ """
+ if status not in ("queued", "running"):
+ return False
+ if isinstance(updated_at, bool) or not isinstance(updated_at, (int, float)):
+ return True
+ return now - updated_at > orphan_after
diff --git a/workers/model-trellis/main.py b/workers/model-trellis/main.py
index 78199077f..a25446c0a 100644
--- a/workers/model-trellis/main.py
+++ b/workers/model-trellis/main.py
@@ -67,6 +67,8 @@ import io
import json
import logging
import os
+import shutil
+import signal
import time
import uuid
@@ -106,6 +108,14 @@ from worker_security import (
safe_error,
)
from storage_backend import LocalStorage
+from instance_health import (
+ HEARTBEAT_SECS,
+ MemorySample,
+ read_memory,
+ should_recycle,
+ task_is_orphaned,
+ trim_heap,
+)
logging.basicConfig(
level=logging.INFO,
@@ -168,6 +178,22 @@ _dinov2_pinned = False
# dict just avoids a GCS round-trip when a poll happens to hit the same warm
# instance that ran the job.
_tasks: dict[str, dict] = {}
+# One lock per task serializes its durable writes. The heartbeat and the status
+# transitions both persist the same record from different coroutines, and the
+# uploads finish on executor threads in any order, so without this a heartbeat
+# that started before a "done" write could land after it and resurrect the task
+# as "running".
+_task_locks: dict[str, asyncio.Lock] = {}
+# Tasks this instance has accepted and not yet finished (queued behind the
+# semaphore or running). The recycle guard only restarts an idle instance.
+_live_tasks: set[str] = set()
+# Memory held right after the model finished loading: the reference the recycle
+# guard measures growth against (see instance_health.should_recycle).
+_memory_baseline: Optional[MemorySample] = None
+# Set once this instance has decided to restart between jobs; /infer then refuses
+# new work with a 503 so the caller fails over instead of queueing behind it.
+_draining = False
+_restart_signalled = False
def _stage_weights_local() -> Optional[str]:
@@ -253,7 +279,7 @@ def _pin_dinov2_to_local_checkout() -> bool:
def _load_pipeline():
- global _pipeline
+ global _pipeline, _memory_baseline
from trellis.pipelines import TrellisImageTo3DPipeline
weights_path = _stage_weights_local() or WEIGHTS_DIR
@@ -262,6 +288,24 @@ def _load_pipeline():
# TRELLIS exposes .cuda() (not .to()) to move every sub-model to the GPU.
_pipeline.cuda()
log.info("TRELLIS pipeline loaded")
+ if weights_path == WEIGHTS_LOCAL_DIR:
+ _release_staged_weights()
+ trim_heap()
+ _memory_baseline = read_memory()
+ if _memory_baseline is not None:
+ log.info("memory after load: %s", _memory_baseline.describe())
+
+
+def _release_staged_weights() -> None:
+ """Delete the staged weight tree once every sub-model is on the GPU.
+
+ Cloud Run's writable filesystem lives in memory, so the 3 GiB staged copy in
+ WEIGHTS_LOCAL_DIR is charged against the same 32 GiB limit the jobs need for
+ as long as it exists. from_pretrained reads every file eagerly, so after the
+ load nothing touches it again, and a later load retry re-stages from GCS.
+ """
+ shutil.rmtree(WEIGHTS_LOCAL_DIR, ignore_errors=True)
+ log.info("released staged weights at %s", WEIGHTS_LOCAL_DIR)
async def _load_pipeline_bg():
@@ -481,29 +525,52 @@ def _get_local_storage() -> LocalStorage:
async def _update_task(task_id: str, **fields) -> dict:
"""Merge `fields` into the task, then persist to GCS as the source of truth
- (see the comment on `_tasks`)."""
- task = _tasks.setdefault(task_id, {"task_id": task_id})
- task.update(fields)
- task["updated_at"] = time.time()
- loop = asyncio.get_event_loop()
- if _using_gcs():
- await loop.run_in_executor(
- None,
- lambda: _task_blob(task_id).upload_from_string(
- json.dumps(task), content_type="application/json"
- ),
- )
- else:
- await loop.run_in_executor(None, _get_local_storage().write_task, task.copy())
+ (see the comment on `_tasks`). Called with no fields it is a heartbeat: it
+ only refreshes updated_at, which is what proves the runner is alive."""
+ lock = _task_locks.setdefault(task_id, asyncio.Lock())
+ async with lock:
+ task = _tasks.setdefault(task_id, {"task_id": task_id})
+ task.update(fields)
+ task["updated_at"] = time.time()
+ # Snapshot inside the lock: the upload runs later on an executor thread,
+ # and serializing the live dict there could capture a newer state than
+ # the one this write is ordered as.
+ snapshot = task.copy()
+ loop = asyncio.get_event_loop()
+ if _using_gcs():
+ payload = json.dumps(snapshot)
+ await loop.run_in_executor(
+ None,
+ lambda: _task_blob(task_id).upload_from_string(
+ payload, content_type="application/json"
+ ),
+ )
+ else:
+ await loop.run_in_executor(None, _get_local_storage().write_task, snapshot)
return task
-# A queued/running record with no state transition for this long is orphaned:
-# its runner instance died (or lost the background task) and nothing resumes
-# persisted tasks. The ceiling covers the 900s pipeline-ready wait plus the
-# longest real generation with margin; expiring it turns an endless client
-# poll into a designed failure the router's poll-time failover can act on.
-_PENDING_TTL_SECS = 1800
+async def _heartbeat(task_id: str) -> None:
+ """Refresh a live task's durable record until cancelled.
+
+ The poll side (_resolve_task) declares a queued/running record orphaned once
+ it stops moving, so this beat is the runner's proof of life. A failed beat
+ is logged and retried on the next tick; it never touches the job itself.
+ """
+ while True:
+ await asyncio.sleep(HEARTBEAT_SECS)
+ try:
+ await _update_task(task_id)
+ except Exception as exc: # noqa: BLE001 - a missed beat must not kill the job
+ log.warning("[%s] heartbeat write failed: %s", task_id, exc)
+
+
+# A live runner refreshes its record every HEARTBEAT_SECS (see _heartbeat), so a
+# queued/running record that stops moving belongs to an instance that died
+# mid-job (an OOM kill, a crash, a scale-in) and nothing resumes persisted tasks.
+# instance_health.ORPHAN_AFTER_SECS bounds that silence; expiring the record
+# turns an endless client poll into a designed failure the router's poll-time
+# failover can redispatch while the client is still polling.
_TERMINAL_STATUSES = frozenset({"done", "failed"})
@@ -537,20 +604,18 @@ async def _resolve_task(task_id: str) -> dict:
if status in _TERMINAL_STATUSES:
_tasks[task_id] = task
return task
- # Records written before updated_at existed can't prove liveness; after a
- # deploy their runner instances are gone, so treat them as orphaned too.
- updated_at = task.get("updated_at")
- stale = (
- not isinstance(updated_at, (int, float))
- or time.time() - updated_at > _PENDING_TTL_SECS
- )
- if status in ("queued", "running") and stale:
- return await _update_task(
+ if task_is_orphaned(status, task.get("updated_at"), time.time()):
+ # Adopt the durable copy so the failure write keeps its fields rather
+ # than whatever partial view this instance had cached.
+ _tasks[task_id] = task
+ failed = await _update_task(
task_id,
status="failed",
- error="task orphaned: no progress within 30 minutes "
- "(runner instance likely restarted mid-job); retry the request",
+ error="task orphaned: its runner instance stopped mid-job "
+ "(restart or out of memory); retry the request",
)
+ _task_locks.pop(task_id, None)
+ return failed
return task
@@ -559,7 +624,58 @@ async def _resolve_task(task_id: str) -> dict:
# CUDA, or the TRELLIS source tree (see test_request_policy.py).
-async def _run_inference(
+async def _run_inference(task_id: str, *args, **kwargs) -> None:
+ """Run one accepted job with a heartbeat on its record, then decide whether
+ this instance should restart before it takes the next one."""
+ _live_tasks.add(task_id)
+ beat = asyncio.create_task(_heartbeat(task_id))
+ try:
+ await _run_job(task_id, *args, **kwargs)
+ finally:
+ beat.cancel()
+ _live_tasks.discard(task_id)
+ _task_locks.pop(task_id, None)
+ _recycle_if_bloated(task_id)
+
+
+def _recycle_if_bloated(task_id: str) -> None:
+ """Restart an idle instance whose memory has ratcheted toward its ceiling.
+
+ Even with the heap trimmed after every job, the native libraries in the
+ postprocess (xatlas, nvdiffrast, open3d) keep some of what they allocate,
+ and this instance can only ever have 32 GiB. Letting it run until a job
+ tips it over is what used to happen: the OOM kill took that job and the one
+ queued behind it. Restarting while idle costs one model load instead.
+
+ Once it decides, the instance drains: /infer refuses new work with a 503 so
+ the caller fails over, jobs already queued here still run, and the restart
+ happens when the last of them finishes. Runs on the event loop with no
+ await, so nothing can be accepted between the idle check and the signal.
+ uvicorn shuts down cleanly on SIGTERM, and Cloud Run starts a fresh instance
+ in its place to hold the warm minimum.
+ """
+ global _draining, _restart_signalled
+ sample = read_memory()
+ if sample is not None:
+ log.info("[%s] memory after job: %s", task_id, sample.describe())
+ if not _draining:
+ if not should_recycle(sample, _memory_baseline):
+ return
+ _draining = True
+ log.warning(
+ "recycling instance: memory %s, up from %s after load; draining %d queued job(s) first",
+ sample.describe(),
+ _memory_baseline.describe() if _memory_baseline else "unknown",
+ len(_live_tasks),
+ )
+ if _live_tasks or _restart_signalled:
+ return
+ _restart_signalled = True
+ log.warning("recycling instance: idle, restarting now")
+ signal.raise_signal(signal.SIGTERM)
+
+
+async def _run_job(
task_id: str,
images: list[str],
body_type: str,
@@ -740,6 +856,10 @@ def _release_gpu_memory() -> None:
if torch.cuda.is_available():
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
+ # The host side leaks the same way: each executor thread's glibc arena keeps
+ # the job's freed heap, which is how the instance ratcheted into OOM kills
+ # (see instance_health). Hand it back to the kernel.
+ trim_heap()
class InferRequest(BaseModel):
@@ -784,6 +904,13 @@ async def infer(
status_code=503,
detail=f"pipeline unavailable: {_load_error}",
)
+ if _draining:
+ # Same failover signal: this instance is restarting to shed memory (see
+ # _recycle_if_bloated) and would kill a job accepted now.
+ raise HTTPException(
+ status_code=503,
+ detail="instance restarting to reclaim memory; retry shortly",
+ )
task_id = str(uuid.uuid4())
# Persist the "queued" record before responding — a poll can reach a
# different instance than this one the moment the 202 lands, and that
@@ -874,6 +1001,7 @@ async def health(response: Response) -> dict:
"ready": bool(_ready and _ready.is_set()),
"load_error": _load_error,
"load_attempts": _load_attempts,
+ "draining": _draining,
"tiers": list(TIER_PRESETS),
"default_quality": QUALITY_DEFAULTS,
"rembg_matte": bool(REMBG_SERVICE_URL),
diff --git a/workers/model-trellis/test_instance_health.py b/workers/model-trellis/test_instance_health.py
new file mode 100644
index 000000000..1fa83a9ae
--- /dev/null
+++ b/workers/model-trellis/test_instance_health.py
@@ -0,0 +1,142 @@
+"""Unit tests for instance self-care: memory reading, recycle, orphan expiry.
+
+Pure stdlib, no torch and no CUDA, so this runs anywhere:
+
+ cd workers/model-trellis && python3 -m pytest test_instance_health.py -q
+
+These decisions run after every generation on the production instance. A wrong
+memory reading either restarts a healthy instance after every job or never
+restarts a bloated one, and a wrong orphan rule either fails live jobs or leaves
+dead ones polling for half an hour, which is the outage this module fixed.
+"""
+
+from pathlib import Path
+
+from instance_health import (
+ HEARTBEAT_SECS,
+ ORPHAN_AFTER_SECS,
+ MemorySample,
+ read_memory,
+ should_recycle,
+ task_is_orphaned,
+ trim_heap,
+)
+
+GIB = 1024 ** 3
+
+
+def _write(path: Path, text: str) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text, encoding="ascii")
+
+
+# ---------------------------------------------------------------- read_memory
+
+
+def test_cgroup_v2_counts_heap_and_tmpfs_but_not_page_cache(tmp_path):
+ v2 = tmp_path / "v2"
+ _write(v2 / "memory.stat", f"anon {20 * GIB}\nfile {9 * GIB}\nshmem {3 * GIB}\n")
+ _write(v2 / "memory.max", f"{32 * GIB}\n")
+ sample = read_memory(str(v2), str(tmp_path / "none"), str(tmp_path / "none"))
+ assert sample == MemorySample(23 * GIB, 32 * GIB, "cgroup2")
+ assert abs(sample.fraction - 23 / 32) < 1e-9
+
+
+def test_cgroup_v2_without_a_limit_falls_through_to_meminfo(tmp_path):
+ v2 = tmp_path / "v2"
+ _write(v2 / "memory.stat", f"anon {GIB}\n")
+ _write(v2 / "memory.max", "max\n")
+ meminfo = tmp_path / "meminfo"
+ _write(meminfo, "MemTotal: 33554432 kB\nMemAvailable: 8388608 kB\n")
+ sample = read_memory(str(v2), str(tmp_path / "none"), str(meminfo))
+ assert sample.source == "meminfo"
+ assert sample.used_bytes == 24 * GIB
+ assert sample.limit_bytes == 32 * GIB
+
+
+def test_cgroup_v1_reads_rss_plus_shmem(tmp_path):
+ v1 = tmp_path / "v1"
+ _write(v1 / "memory.stat", f"total_rss {10 * GIB}\ntotal_cache {5 * GIB}\ntotal_shmem {2 * GIB}\n")
+ _write(v1 / "memory.limit_in_bytes", f"{16 * GIB}\n")
+ sample = read_memory(str(tmp_path / "none"), str(v1), str(tmp_path / "none"))
+ assert sample == MemorySample(12 * GIB, 16 * GIB, "cgroup1")
+
+
+def test_unlimited_cgroup_v1_is_not_a_ceiling(tmp_path):
+ v1 = tmp_path / "v1"
+ _write(v1 / "memory.stat", f"total_rss {GIB}\n")
+ _write(v1 / "memory.limit_in_bytes", "9223372036854771712\n")
+ assert read_memory(str(tmp_path / "none"), str(v1), str(tmp_path / "none")) is None
+
+
+def test_nothing_readable_is_unknown_not_an_error(tmp_path):
+ missing = str(tmp_path / "missing")
+ assert read_memory(missing, missing, missing) is None
+
+
+def test_describe_is_human_readable():
+ text = MemorySample(24 * GIB, 32 * GIB, "cgroup2").describe()
+ assert text == "24.0/32.0 GiB (75%, cgroup2)"
+
+
+# ---------------------------------------------------------------- should_recycle
+
+
+def _at(fraction: float) -> MemorySample:
+ return MemorySample(int(fraction * 32 * GIB), 32 * GIB, "cgroup2")
+
+
+def test_recycles_a_ratcheted_instance():
+ assert should_recycle(_at(0.88), _at(0.46))
+
+
+def test_leaves_a_healthy_instance_alone():
+ assert not should_recycle(_at(0.60), _at(0.46))
+
+
+def test_high_baseline_alone_never_recycles():
+ # A baseline that already reads high (a big model, or a reading that counts
+ # more than it should) must not restart the instance after every job.
+ assert not should_recycle(_at(0.84), _at(0.80))
+
+
+def test_unknown_memory_never_recycles():
+ assert not should_recycle(None, _at(0.40))
+ assert not should_recycle(_at(0.95), None)
+
+
+# ---------------------------------------------------------------- task_is_orphaned
+
+
+NOW = 1_800_000_000.0
+
+
+def test_a_beating_task_is_alive():
+ assert not task_is_orphaned("running", NOW - HEARTBEAT_SECS, NOW)
+ assert not task_is_orphaned("queued", NOW - ORPHAN_AFTER_SECS + 1, NOW)
+
+
+def test_a_silent_task_is_orphaned_within_minutes():
+ assert task_is_orphaned("running", NOW - ORPHAN_AFTER_SECS - 1, NOW)
+ # The old rule waited 1800 s; the whole point is that a client still polls.
+ assert ORPHAN_AFTER_SECS <= 300
+ # Several missed beats, never one, decide it.
+ assert ORPHAN_AFTER_SECS >= 4 * HEARTBEAT_SECS
+
+
+def test_a_record_without_a_timestamp_cannot_prove_it_is_alive():
+ assert task_is_orphaned("queued", None, NOW)
+ assert task_is_orphaned("running", "yesterday", NOW)
+ assert task_is_orphaned("running", True, NOW)
+
+
+def test_terminal_and_unknown_records_are_never_orphaned():
+ for status in ("done", "failed", None, "weird"):
+ assert not task_is_orphaned(status, NOW - 10 * ORPHAN_AFTER_SECS, NOW)
+
+
+# ---------------------------------------------------------------- trim_heap
+
+
+def test_trim_heap_never_raises():
+ assert trim_heap() in (True, False)