Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(trellis): cap malloc arenas, recycle on memory pressure and expire orphaned tasks
nirholas7f0f36e71c5a
diff --git a/workers/model-trellis/Dockerfile b/workers/model-trellis/Dockerfileindex 1eab909c0..d5ba7094a 100644--- a/workers/model-trellis/Dockerfile+++ b/workers/model-trellis/Dockerfile@@ -25,7 +25,12 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ATTN_BACKEND=xformers \SPCONV_ALGO=native \# TRELLIS repo lives here and is imported off PYTHONPATH (not a pip package).- PYTHONPATH=/app/TRELLIS+ PYTHONPATH=/app/TRELLIS \+ # Cap glibc at two malloc arenas. Every job runs on executor threads, and by+ # default each thread grows its own arena that keeps freed heap forever: the+ # instance ratcheted from 46% to 91% of its 32 GiB across a day of jobs and+ # was OOM-killed mid-job (see instance_health.py).+ MALLOC_ARENA_MAX=2# libusb-1.0-0 and libegl1: open3d (imported by TRELLIS's pipelines package)# dlopens both at import time. Without them the model load dies with@@ -125,17 +130,19 @@ WORKDIR /appCOPY worker_security.py /app/worker_security.pyCOPY request_policy.py /app/request_policy.pyCOPY storage_backend.py /app/storage_backend.py+COPY instance_health.py /app/instance_health.pyCOPY main.py /app/main.py-COPY test_request_policy.py test_storage_backend.py test_app_contract.py /app/+COPY test_request_policy.py test_storage_backend.py test_instance_health.py test_app_contract.py /app/# Build-time gate, neither half of which needs a GPU:# 1. The quality tiers, the clamps that keep a caller value from crashing the-# nvdiffrast bake, and the image-fetch retry policy are pure logic, proven-# here rather than in production.+# nvdiffrast bake, the image-fetch retry policy, and the memory-recycle and+# orphan-expiry decisions are pure logic, proven here rather than in+# production.# 2. The served contract, against the exact dependency set that will run: the# TRELLIS import chain (FlexiCubes submodule included), the auth boundary,# the probe surfaces, request validation, and the SSRF guard.-RUN python3 -m pytest /app/test_request_policy.py /app/test_storage_backend.py -q \+RUN python3 -m pytest /app/test_request_policy.py /app/test_storage_backend.py /app/test_instance_health.py -q \&& python3 /app/test_app_contract.pyEXPOSE 8080diff --git a/workers/model-trellis/instance_health.py b/workers/model-trellis/instance_health.pynew file mode 100644index 000000000..124dbc40a--- /dev/null+++ b/workers/model-trellis/instance_health.py@@ -0,0 +1,219 @@+"""Instance self-care for the TRELLIS worker: host memory and task liveness.++Pure stdlib, no torch and no CUDA, so every decision here is unit tested on any+machine (see test_instance_health.py) and main.py only wires it in.++Why this module exists (production, 2026-10-03 to 2026-10-08):++ * The single warm L4 instance was OOM-killed twelve times in five days+ ("Container terminated on signal 9"). Cloud Run's memory utilization for+ the revision climbed in steps after each heavy job (0.46 -> 0.56 -> 0.71 ->+ 0.81 -> 0.88 -> 0.91) and never came back down, then a mesh postprocess+ (decimate, invisible-face removal, UV unwrap, texture bake) pushed it past+ the 32 GiB ceiling, which is the most memory Cloud Run gives an L4 instance.+ That is glibc keeping freed heap in per-thread arenas: the pipeline runs in+ executor threads, each of which grows its own arena, and nothing hands the+ freed pages back to the kernel.+ * Every kill took the in-flight job with it, and the queued job behind it.+ Their durable GCS records stayed "running" until a 30 minute orphan window+ expired, long after the caller's poll budget, so the forge's lane failover+ never saw a failure to act on and the user just watched a spinner.++The fixes are split by what they guard against:++ trim_heap() returns freed heap to the kernel after every job.+ read_memory() reports what the instance actually holds, so each job+ logs its footprint and the recycle guard has a number.+ should_recycle() decides when an instance has ratcheted far enough that+ it should restart BETWEEN jobs rather than die inside one.+ task_is_orphaned() lets a heartbeat-refreshed record expire in minutes, not+ half an hour, once its runner is gone.+"""++from __future__ import annotations++import ctypes+import ctypes.util+import os+from dataclasses import dataclass+from typing import Optional++# How often a live task refreshes its durable record, and how long a queued or+# running record may go without a refresh before it is declared orphaned. Six+# missed beats is far beyond any scheduling hiccup on a healthy instance, and+# still well inside the forge client's ~300 s poll budget, so the failure lands+# while a client is still listening and the lane failover can redispatch.+HEARTBEAT_SECS = 30.0+ORPHAN_AFTER_SECS = 180.0++# Recycle once the instance holds this share of its memory limit after a job...+RECYCLE_FRACTION = float(os.environ.get("MEMORY_RECYCLE_FRACTION", "0.80"))+# ...and only if it has grown at least this much past its post-load baseline. The+# growth floor is what stops a mismeasured or simply large baseline from turning+# into a restart after every single job.+RECYCLE_MIN_GROWTH = float(os.environ.get("MEMORY_RECYCLE_MIN_GROWTH", "0.15"))++_CGROUP_V2 = "/sys/fs/cgroup"+_CGROUP_V1 = "/sys/fs/cgroup/memory"+_MEMINFO = "/proc/meminfo"+++@dataclass(frozen=True)+class MemorySample:+ used_bytes: int+ limit_bytes: int+ source: str++ @property+ def fraction(self) -> float:+ return self.used_bytes / self.limit_bytes if self.limit_bytes else 0.0++ def describe(self) -> str:+ gib = 1024 ** 3+ return (+ f"{self.used_bytes / gib:.1f}/{self.limit_bytes / gib:.1f} GiB "+ f"({self.fraction:.0%}, {self.source})"+ )+++def _read(path: str) -> Optional[str]:+ try:+ with open(path, encoding="ascii") as fh:+ return fh.read()+ except OSError:+ return None+++def _stat_fields(text: str) -> dict[str, int]:+ fields: dict[str, int] = {}+ for line in text.splitlines():+ parts = line.split()+ if len(parts) >= 2 and parts[1].isdigit():+ fields[parts[0].rstrip(":")] = int(parts[1])+ return fields+++def _from_cgroup_v2(root: str) -> Optional[MemorySample]:+ stat, limit = _read(os.path.join(root, "memory.stat")), _read(os.path.join(root, "memory.max"))+ if stat is None or limit is None or not limit.strip().isdigit():+ return None+ fields = _stat_fields(stat)+ if "anon" not in fields:+ return None+ # anon is the process heap; shmem is tmpfs, which is where Cloud Run's+ # in-memory filesystem (and so the staged weights in /tmp) is charged.+ # Page cache is left out on purpose: the kernel reclaims it before it OOMs.+ return MemorySample(fields["anon"] + fields.get("shmem", 0), int(limit.strip()), "cgroup2")+++def _from_cgroup_v1(root: str) -> Optional[MemorySample]:+ stat = _read(os.path.join(root, "memory.stat"))+ limit = _read(os.path.join(root, "memory.limit_in_bytes"))+ if stat is None or limit is None or not limit.strip().isdigit():+ return None+ fields = _stat_fields(stat)+ if "total_rss" not in fields:+ return None+ limit_bytes = int(limit.strip())+ # An unlimited v1 group reports a sentinel near 2**63; it says nothing about+ # this instance's ceiling, so defer to /proc/meminfo instead.+ if limit_bytes >= 1 << 60:+ return None+ return MemorySample(fields["total_rss"] + fields.get("total_shmem", 0), limit_bytes, "cgroup1")+++def _from_meminfo(path: str) -> Optional[MemorySample]:+ text = _read(path)+ if text is None:+ return None+ fields = _stat_fields(text)+ total, available = fields.get("MemTotal"), fields.get("MemAvailable")+ if not total or available is None:+ return None+ # /proc/meminfo is in KiB. MemAvailable already discounts reclaimable cache.+ return MemorySample((total - available) * 1024, total * 1024, "meminfo")+++def read_memory(+ cgroup_v2_root: str = _CGROUP_V2,+ cgroup_v1_root: str = _CGROUP_V1,+ meminfo_path: str = _MEMINFO,+) -> Optional[MemorySample]:+ """Best available view of the memory this instance holds against its limit.++ Never raises. Returns None when no source is readable, and every caller+ treats None as "unknown", which never triggers a recycle.+ """+ for reader, arg in (+ (_from_cgroup_v2, cgroup_v2_root),+ (_from_cgroup_v1, cgroup_v1_root),+ (_from_meminfo, meminfo_path),+ ):+ try:+ sample = reader(arg)+ except (ValueError, ZeroDivisionError):+ sample = None+ if sample is not None and sample.limit_bytes > 0:+ return sample+ return None+++def should_recycle(+ current: Optional[MemorySample],+ baseline: Optional[MemorySample],+ threshold: float = RECYCLE_FRACTION,+ min_growth: float = RECYCLE_MIN_GROWTH,+) -> bool:+ """True when the instance should restart between jobs.++ Both conditions must hold: the instance is near its ceiling, AND it got+ there by growing since the model finished loading. Without a baseline (the+ load has not completed, or memory is unreadable) the answer is always no.+ """+ if current is None or baseline is None:+ return False+ return current.fraction >= threshold and current.fraction - baseline.fraction >= min_growth+++_libc = None+++def trim_heap() -> bool:+ """Hand freed heap pages back to the kernel (glibc malloc_trim).++ Returns False on a libc without malloc_trim (musl, macOS), where there is+ nothing to do. Never raises.+ """+ global _libc+ try:+ if _libc is None:+ _libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6")+ trim = getattr(_libc, "malloc_trim", None)+ if trim is None:+ return False+ trim.argtypes = [ctypes.c_size_t]+ trim.restype = ctypes.c_int+ trim(0)+ return True+ except OSError:+ return False+++def task_is_orphaned(+ status: Optional[str],+ updated_at: object,+ now: float,+ orphan_after: float = ORPHAN_AFTER_SECS,+) -> bool:+ """True when a queued or running record has gone stale.++ A live runner refreshes updated_at every HEARTBEAT_SECS, so a record that+ has not moved for orphan_after seconds belongs to an instance that died.+ A record with no usable timestamp cannot prove it is alive, so it counts as+ orphaned too. Terminal and unknown statuses never are.+ """+ if status not in ("queued", "running"):+ return False+ if isinstance(updated_at, bool) or not isinstance(updated_at, (int, float)):+ return True+ return now - updated_at > orphan_afterdiff --git a/workers/model-trellis/main.py b/workers/model-trellis/main.pyindex 78199077f..a25446c0a 100644--- a/workers/model-trellis/main.py+++ b/workers/model-trellis/main.py@@ -67,6 +67,8 @@ import ioimport jsonimport loggingimport os+import shutil+import signalimport timeimport uuid@@ -106,6 +108,14 @@ from worker_security import (safe_error,)from storage_backend import LocalStorage+from instance_health import (+ HEARTBEAT_SECS,+ MemorySample,+ read_memory,+ should_recycle,+ task_is_orphaned,+ trim_heap,+)logging.basicConfig(level=logging.INFO,@@ -168,6 +178,22 @@ _dinov2_pinned = False# dict just avoids a GCS round-trip when a poll happens to hit the same warm# instance that ran the job._tasks: dict[str, dict] = {}+# One lock per task serializes its durable writes. The heartbeat and the status+# transitions both persist the same record from different coroutines, and the+# uploads finish on executor threads in any order, so without this a heartbeat+# that started before a "done" write could land after it and resurrect the task+# as "running".+_task_locks: dict[str, asyncio.Lock] = {}+# Tasks this instance has accepted and not yet finished (queued behind the+# semaphore or running). The recycle guard only restarts an idle instance.+_live_tasks: set[str] = set()+# Memory held right after the model finished loading: the reference the recycle+# guard measures growth against (see instance_health.should_recycle).+_memory_baseline: Optional[MemorySample] = None+# Set once this instance has decided to restart between jobs; /infer then refuses+# new work with a 503 so the caller fails over instead of queueing behind it.+_draining = False+_restart_signalled = Falsedef _stage_weights_local() -> Optional[str]:@@ -253,7 +279,7 @@ def _pin_dinov2_to_local_checkout() -> bool:def _load_pipeline():- global _pipeline+ global _pipeline, _memory_baselinefrom trellis.pipelines import TrellisImageTo3DPipelineweights_path = _stage_weights_local() or WEIGHTS_DIR@@ -262,6 +288,24 @@ def _load_pipeline():# TRELLIS exposes .cuda() (not .to()) to move every sub-model to the GPU._pipeline.cuda()log.info("TRELLIS pipeline loaded")+ if weights_path == WEIGHTS_LOCAL_DIR:+ _release_staged_weights()+ trim_heap()+ _memory_baseline = read_memory()+ if _memory_baseline is not None:+ log.info("memory after load: %s", _memory_baseline.describe())+++def _release_staged_weights() -> None:+ """Delete the staged weight tree once every sub-model is on the GPU.++ Cloud Run's writable filesystem lives in memory, so the 3 GiB staged copy in+ WEIGHTS_LOCAL_DIR is charged against the same 32 GiB limit the jobs need for+ as long as it exists. from_pretrained reads every file eagerly, so after the+ load nothing touches it again, and a later load retry re-stages from GCS.+ """+ shutil.rmtree(WEIGHTS_LOCAL_DIR, ignore_errors=True)+ log.info("released staged weights at %s", WEIGHTS_LOCAL_DIR)async def _load_pipeline_bg():@@ -481,29 +525,52 @@ def _get_local_storage() -> LocalStorage:async def _update_task(task_id: str, **fields) -> dict:"""Merge `fields` into the task, then persist to GCS as the source of truth- (see the comment on `_tasks`)."""- task = _tasks.setdefault(task_id, {"task_id": task_id})- task.update(fields)- task["updated_at"] = time.time()- loop = asyncio.get_event_loop()- if _using_gcs():- await loop.run_in_executor(- None,- lambda: _task_blob(task_id).upload_from_string(- json.dumps(task), content_type="application/json"- ),- )- else:- await loop.run_in_executor(None, _get_local_storage().write_task, task.copy())+ (see the comment on `_tasks`). Called with no fields it is a heartbeat: it+ only refreshes updated_at, which is what proves the runner is alive."""+ lock = _task_locks.setdefault(task_id, asyncio.Lock())+ async with lock:+ task = _tasks.setdefault(task_id, {"task_id": task_id})+ task.update(fields)+ task["updated_at"] = time.time()+ # Snapshot inside the lock: the upload runs later on an executor thread,+ # and serializing the live dict there could capture a newer state than+ # the one this write is ordered as.+ snapshot = task.copy()+ loop = asyncio.get_event_loop()+ if _using_gcs():+ payload = json.dumps(snapshot)+ await loop.run_in_executor(+ None,+ lambda: _task_blob(task_id).upload_from_string(+ payload, content_type="application/json"+ ),+ )+ else:+ await loop.run_in_executor(None, _get_local_storage().write_task, snapshot)return task-# A queued/running record with no state transition for this long is orphaned:-# its runner instance died (or lost the background task) and nothing resumes-# persisted tasks. The ceiling covers the 900s pipeline-ready wait plus the-# longest real generation with margin; expiring it turns an endless client-# poll into a designed failure the router's poll-time failover can act on.-_PENDING_TTL_SECS = 1800+async def _heartbeat(task_id: str) -> None:+ """Refresh a live task's durable record until cancelled.++ The poll side (_resolve_task) declares a queued/running record orphaned once+ it stops moving, so this beat is the runner's proof of life. A failed beat+ is logged and retried on the next tick; it never touches the job itself.+ """+ while True:+ await asyncio.sleep(HEARTBEAT_SECS)+ try:+ await _update_task(task_id)+ except Exception as exc: # noqa: BLE001 - a missed beat must not kill the job+ log.warning("[%s] heartbeat write failed: %s", task_id, exc)+++# A live runner refreshes its record every HEARTBEAT_SECS (see _heartbeat), so a+# queued/running record that stops moving belongs to an instance that died+# mid-job (an OOM kill, a crash, a scale-in) and nothing resumes persisted tasks.+# instance_health.ORPHAN_AFTER_SECS bounds that silence; expiring the record+# turns an endless client poll into a designed failure the router's poll-time+# failover can redispatch while the client is still polling._TERMINAL_STATUSES = frozenset({"done", "failed"})@@ -537,20 +604,18 @@ async def _resolve_task(task_id: str) -> dict:if status in _TERMINAL_STATUSES:_tasks[task_id] = taskreturn task- # Records written before updated_at existed can't prove liveness; after a- # deploy their runner instances are gone, so treat them as orphaned too.- updated_at = task.get("updated_at")- stale = (- not isinstance(updated_at, (int, float))- or time.time() - updated_at > _PENDING_TTL_SECS- )- if status in ("queued", "running") and stale:- return await _update_task(+ if task_is_orphaned(status, task.get("updated_at"), time.time()):+ # Adopt the durable copy so the failure write keeps its fields rather+ # than whatever partial view this instance had cached.+ _tasks[task_id] = task+ failed = await _update_task(task_id,status="failed",- error="task orphaned: no progress within 30 minutes "- "(runner instance likely restarted mid-job); retry the request",+ error="task orphaned: its runner instance stopped mid-job "+ "(restart or out of memory); retry the request",)+ _task_locks.pop(task_id, None)+ return failedreturn task@@ -559,7 +624,58 @@ async def _resolve_task(task_id: str) -> dict:# CUDA, or the TRELLIS source tree (see test_request_policy.py).-async def _run_inference(+async def _run_inference(task_id: str, *args, **kwargs) -> None:+ """Run one accepted job with a heartbeat on its record, then decide whether+ this instance should restart before it takes the next one."""+ _live_tasks.add(task_id)+ beat = asyncio.create_task(_heartbeat(task_id))+ try:+ await _run_job(task_id, *args, **kwargs)+ finally:+ beat.cancel()+ _live_tasks.discard(task_id)+ _task_locks.pop(task_id, None)+ _recycle_if_bloated(task_id)+++def _recycle_if_bloated(task_id: str) -> None:+ """Restart an idle instance whose memory has ratcheted toward its ceiling.++ Even with the heap trimmed after every job, the native libraries in the+ postprocess (xatlas, nvdiffrast, open3d) keep some of what they allocate,+ and this instance can only ever have 32 GiB. Letting it run until a job+ tips it over is what used to happen: the OOM kill took that job and the one+ queued behind it. Restarting while idle costs one model load instead.++ Once it decides, the instance drains: /infer refuses new work with a 503 so+ the caller fails over, jobs already queued here still run, and the restart+ happens when the last of them finishes. Runs on the event loop with no+ await, so nothing can be accepted between the idle check and the signal.+ uvicorn shuts down cleanly on SIGTERM, and Cloud Run starts a fresh instance+ in its place to hold the warm minimum.+ """+ global _draining, _restart_signalled+ sample = read_memory()+ if sample is not None:+ log.info("[%s] memory after job: %s", task_id, sample.describe())+ if not _draining:+ if not should_recycle(sample, _memory_baseline):+ return+ _draining = True+ log.warning(+ "recycling instance: memory %s, up from %s after load; draining %d queued job(s) first",+ sample.describe(),+ _memory_baseline.describe() if _memory_baseline else "unknown",+ len(_live_tasks),+ )+ if _live_tasks or _restart_signalled:+ return+ _restart_signalled = True+ log.warning("recycling instance: idle, restarting now")+ signal.raise_signal(signal.SIGTERM)+++async def _run_job(task_id: str,images: list[str],body_type: str,@@ -740,6 +856,10 @@ def _release_gpu_memory() -> None:if torch.cuda.is_available():torch.cuda.empty_cache()torch.cuda.ipc_collect()+ # The host side leaks the same way: each executor thread's glibc arena keeps+ # the job's freed heap, which is how the instance ratcheted into OOM kills+ # (see instance_health). Hand it back to the kernel.+ trim_heap()class InferRequest(BaseModel):@@ -784,6 +904,13 @@ async def infer(status_code=503,detail=f"pipeline unavailable: {_load_error}",)+ if _draining:+ # Same failover signal: this instance is restarting to shed memory (see+ # _recycle_if_bloated) and would kill a job accepted now.+ raise HTTPException(+ status_code=503,+ detail="instance restarting to reclaim memory; retry shortly",+ )task_id = str(uuid.uuid4())# Persist the "queued" record before responding — a poll can reach a# different instance than this one the moment the 202 lands, and that@@ -874,6 +1001,7 @@ async def health(response: Response) -> dict:"ready": bool(_ready and _ready.is_set()),"load_error": _load_error,"load_attempts": _load_attempts,+ "draining": _draining,"tiers": list(TIER_PRESETS),"default_quality": QUALITY_DEFAULTS,"rembg_matte": bool(REMBG_SERVICE_URL),diff --git a/workers/model-trellis/test_instance_health.py b/workers/model-trellis/test_instance_health.pynew file mode 100644index 000000000..1fa83a9ae--- /dev/null+++ b/workers/model-trellis/test_instance_health.py@@ -0,0 +1,142 @@+"""Unit tests for instance self-care: memory reading, recycle, orphan expiry.++Pure stdlib, no torch and no CUDA, so this runs anywhere:++ cd workers/model-trellis && python3 -m pytest test_instance_health.py -q++These decisions run after every generation on the production instance. A wrong+memory reading either restarts a healthy instance after every job or never+restarts a bloated one, and a wrong orphan rule either fails live jobs or leaves+dead ones polling for half an hour, which is the outage this module fixed.+"""++from pathlib import Path++from instance_health import (+ HEARTBEAT_SECS,+ ORPHAN_AFTER_SECS,+ MemorySample,+ read_memory,+ should_recycle,+ task_is_orphaned,+ trim_heap,+)++GIB = 1024 ** 3+++def _write(path: Path, text: str) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(text, encoding="ascii")+++# ---------------------------------------------------------------- read_memory+++def test_cgroup_v2_counts_heap_and_tmpfs_but_not_page_cache(tmp_path):+ v2 = tmp_path / "v2"+ _write(v2 / "memory.stat", f"anon {20 * GIB}\nfile {9 * GIB}\nshmem {3 * GIB}\n")+ _write(v2 / "memory.max", f"{32 * GIB}\n")+ sample = read_memory(str(v2), str(tmp_path / "none"), str(tmp_path / "none"))+ assert sample == MemorySample(23 * GIB, 32 * GIB, "cgroup2")+ assert abs(sample.fraction - 23 / 32) < 1e-9+++def test_cgroup_v2_without_a_limit_falls_through_to_meminfo(tmp_path):+ v2 = tmp_path / "v2"+ _write(v2 / "memory.stat", f"anon {GIB}\n")+ _write(v2 / "memory.max", "max\n")+ meminfo = tmp_path / "meminfo"+ _write(meminfo, "MemTotal: 33554432 kB\nMemAvailable: 8388608 kB\n")+ sample = read_memory(str(v2), str(tmp_path / "none"), str(meminfo))+ assert sample.source == "meminfo"+ assert sample.used_bytes == 24 * GIB+ assert sample.limit_bytes == 32 * GIB+++def test_cgroup_v1_reads_rss_plus_shmem(tmp_path):+ v1 = tmp_path / "v1"+ _write(v1 / "memory.stat", f"total_rss {10 * GIB}\ntotal_cache {5 * GIB}\ntotal_shmem {2 * GIB}\n")+ _write(v1 / "memory.limit_in_bytes", f"{16 * GIB}\n")+ sample = read_memory(str(tmp_path / "none"), str(v1), str(tmp_path / "none"))+ assert sample == MemorySample(12 * GIB, 16 * GIB, "cgroup1")+++def test_unlimited_cgroup_v1_is_not_a_ceiling(tmp_path):+ v1 = tmp_path / "v1"+ _write(v1 / "memory.stat", f"total_rss {GIB}\n")+ _write(v1 / "memory.limit_in_bytes", "9223372036854771712\n")+ assert read_memory(str(tmp_path / "none"), str(v1), str(tmp_path / "none")) is None+++def test_nothing_readable_is_unknown_not_an_error(tmp_path):+ missing = str(tmp_path / "missing")+ assert read_memory(missing, missing, missing) is None+++def test_describe_is_human_readable():+ text = MemorySample(24 * GIB, 32 * GIB, "cgroup2").describe()+ assert text == "24.0/32.0 GiB (75%, cgroup2)"+++# ---------------------------------------------------------------- should_recycle+++def _at(fraction: float) -> MemorySample:+ return MemorySample(int(fraction * 32 * GIB), 32 * GIB, "cgroup2")+++def test_recycles_a_ratcheted_instance():+ assert should_recycle(_at(0.88), _at(0.46))+++def test_leaves_a_healthy_instance_alone():+ assert not should_recycle(_at(0.60), _at(0.46))+++def test_high_baseline_alone_never_recycles():+ # A baseline that already reads high (a big model, or a reading that counts+ # more than it should) must not restart the instance after every job.+ assert not should_recycle(_at(0.84), _at(0.80))+++def test_unknown_memory_never_recycles():+ assert not should_recycle(None, _at(0.40))+ assert not should_recycle(_at(0.95), None)+++# ---------------------------------------------------------------- task_is_orphaned+++NOW = 1_800_000_000.0+++def test_a_beating_task_is_alive():+ assert not task_is_orphaned("running", NOW - HEARTBEAT_SECS, NOW)+ assert not task_is_orphaned("queued", NOW - ORPHAN_AFTER_SECS + 1, NOW)+++def test_a_silent_task_is_orphaned_within_minutes():+ assert task_is_orphaned("running", NOW - ORPHAN_AFTER_SECS - 1, NOW)+ # The old rule waited 1800 s; the whole point is that a client still polls.+ assert ORPHAN_AFTER_SECS <= 300+ # Several missed beats, never one, decide it.+ assert ORPHAN_AFTER_SECS >= 4 * HEARTBEAT_SECS+++def test_a_record_without_a_timestamp_cannot_prove_it_is_alive():+ assert task_is_orphaned("queued", None, NOW)+ assert task_is_orphaned("running", "yesterday", NOW)+ assert task_is_orphaned("running", True, NOW)+++def test_terminal_and_unknown_records_are_never_orphaned():+ for status in ("done", "failed", None, "weird"):+ assert not task_is_orphaned(status, NOW - 10 * ORPHAN_AFTER_SECS, NOW)+++# ---------------------------------------------------------------- trim_heap+++def test_trim_heap_never_raises():+ assert trim_heap() in (True, False)