Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

fix(memory): respect the container cgroup memory limit instead of host RAM (#15927)

Alexander Piskun77739723a36e
diff --git a/comfy/model_management.py b/comfy/model_management.py
index ff963eb8e..ffc4f129d 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -30,6 +30,7 @@ import gc
import os
from contextlib import contextmanager, nullcontext
import comfy.memory_management
+import comfy.system_memory
import comfy.utils
import comfy.quant_ops
import comfy_aimdo.host_buffer
@@ -318,7 +319,7 @@ def get_total_memory(dev=None, torch_total_too=False):
dev = get_torch_device()
if hasattr(dev, 'type') and (dev.type == 'cpu' or dev.type == 'mps'):
- mem_total = psutil.virtual_memory().total
+ mem_total = comfy.system_memory.virtual_memory_total()
mem_total_torch = mem_total
else:
if directml_enabled:
@@ -361,8 +362,11 @@ def mac_version():
return None
total_vram = get_total_memory(get_torch_device()) / (1024 * 1024)
-total_ram = psutil.virtual_memory().total / (1024 * 1024)
+total_ram = comfy.system_memory.virtual_memory_total() / (1024 * 1024)
logging.info("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram))
+cgroup_ram_limit = comfy.system_memory.cgroup_memory_limit()
+if cgroup_ram_limit is not None:
+ logging.info("RAM limited by cgroup to {:0.0f} MB (host has {:0.0f} MB)".format(cgroup_ram_limit / (1024 * 1024), psutil.virtual_memory().total / (1024 * 1024)))
try:
logging.info("pytorch version: {}".format(torch_version))
@@ -703,7 +707,7 @@ def should_free_pins_for_ram_pressure(shortfall):
return False
if not WINDOWS:
return True
- if psutil.virtual_memory().available < WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE:
+ if comfy.system_memory.virtual_memory_available() < WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE:
return True
try:
return psutil.swap_memory().percent >= WINDOWS_PIN_EVICTION_SWAP_PERCENT
@@ -717,7 +721,7 @@ def ensure_pin_budget(size, evict_active=False, loaded=False):
if args.fast_disk:
shortfall = TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY
else:
- shortfall = size + max(comfy.memory_management.RAM_CACHE_HEADROOM / 2, 2048 * 1024 ** 2) - psutil.virtual_memory().available
+ shortfall = size + max(comfy.memory_management.RAM_CACHE_HEADROOM / 2, 2048 * 1024 ** 2) - comfy.system_memory.virtual_memory_available()
if shortfall <= 0:
return True
@@ -1584,7 +1588,8 @@ if not args.disable_pinned_memory:
if WINDOWS:
MAX_PINNED_MEMORY = ram * 0.40 # Windows limit is apparently 50%
else:
- MAX_PINNED_MEMORY = max(ram * 0.40, min(ram * 0.90, ram - 4 * 1024 ** 3, ram + get_disk_swap_total() - 16 * 1024 ** 3))
+ swap = 0 if comfy.system_memory.cgroup_memory_limit() is not None else get_disk_swap_total()
+ MAX_PINNED_MEMORY = max(ram * 0.40, min(ram * 0.90, ram - 4 * 1024 ** 3, ram + swap - 16 * 1024 ** 3))
logging.info("Enabled pinned memory {}".format(MAX_PINNED_MEMORY // (1024 * 1024)))
PINNING_ALLOWED_TYPES = set(["Tensor", "Parameter", "QuantizedTensor"])
@@ -1751,7 +1756,7 @@ def get_free_memory(dev=None, torch_free_too=False):
dev = get_torch_device()
if hasattr(dev, 'type') and (dev.type == 'cpu' or dev.type == 'mps'):
- mem_free_total = psutil.virtual_memory().available
+ mem_free_total = comfy.system_memory.virtual_memory_available()
mem_free_torch = mem_free_total
else:
if directml_enabled:
diff --git a/comfy/system_memory.py b/comfy/system_memory.py
new file mode 100644
index 000000000..63de575f8
--- /dev/null
+++ b/comfy/system_memory.py
@@ -0,0 +1,128 @@
+import os
+import sys
+
+import psutil
+
+CGROUP_V2_ROOT = "/sys/fs/cgroup"
+CGROUP_V1_MEMORY_ROOT = "/sys/fs/cgroup/memory"
+PROC_SELF_CGROUP = "/proc/self/cgroup"
+
+_cgroup_dirs = None
+
+
+def _read_text(path):
+ try:
+ with open(path, encoding="utf-8") as f:
+ return f.read().strip()
+ except (OSError, ValueError):
+ return None
+
+
+def _read_int(path):
+ raw = _read_text(path)
+ if raw is None:
+ return None
+ try:
+ return int(raw)
+ except ValueError:
+ return None
+
+
+def _read_stat(path, key):
+ raw = _read_text(path)
+ if raw is None:
+ return None
+ for line in raw.splitlines():
+ parts = line.split()
+ if len(parts) == 2 and parts[0] == key:
+ try:
+ return int(parts[1])
+ except ValueError:
+ return None
+ return None
+
+
+def _lineage(root, path):
+ dirs = [root]
+ for part in path.split("/"):
+ if part:
+ dirs.append(os.path.join(dirs[-1], part))
+ return dirs[::-1]
+
+
+def _own_cgroup_dirs():
+ raw = _read_text(PROC_SELF_CGROUP)
+ if raw is None:
+ return []
+ dirs = []
+ for line in raw.splitlines():
+ parts = line.split(":", 2)
+ if len(parts) != 3:
+ continue
+ controllers, path = parts[1], parts[2]
+ if controllers == "":
+ root = CGROUP_V2_ROOT
+ elif "memory" in controllers.split(","):
+ root = CGROUP_V1_MEMORY_ROOT
+ else:
+ continue
+ for directory in _lineage(root, path):
+ if directory not in dirs:
+ dirs.append(directory)
+ return dirs
+
+
+def _cgroup_directories():
+ global _cgroup_dirs
+ if _cgroup_dirs is None:
+ _cgroup_dirs = _own_cgroup_dirs() or [CGROUP_V2_ROOT, CGROUP_V1_MEMORY_ROOT]
+ return _cgroup_dirs
+
+
+def _limit_in(directory):
+ for name in ("memory.max", "memory.limit_in_bytes"):
+ value = _read_int(os.path.join(directory, name))
+ if value is not None and value > 0:
+ return value
+ return None
+
+
+def _working_set_in(directory):
+ usage = _read_int(os.path.join(directory, "memory.current"))
+ key = "inactive_file"
+ if usage is None:
+ usage = _read_int(os.path.join(directory, "memory.usage_in_bytes"))
+ key = "total_inactive_file"
+ if usage is None:
+ return None
+ inactive_file = _read_stat(os.path.join(directory, "memory.stat"), key) or 0
+ return max(0, usage - inactive_file)
+
+
+def _limited(host_total):
+ if not sys.platform.startswith("linux"):
+ return []
+ limited = []
+ for directory in _cgroup_directories():
+ limit = _limit_in(directory)
+ if limit is not None and limit < host_total:
+ limited.append((limit, directory))
+ return limited
+
+
+def cgroup_memory_limit():
+ return min((limit for limit, _ in _limited(psutil.virtual_memory().total)), default=None)
+
+
+def virtual_memory_total():
+ host = psutil.virtual_memory()
+ return min((limit for limit, _ in _limited(host.total)), default=host.total)
+
+
+def virtual_memory_available():
+ host = psutil.virtual_memory()
+ available = host.available
+ for limit, directory in _limited(host.total):
+ used = _working_set_in(directory)
+ available = min(available, limit if used is None else limit - used)
+ return max(0, available)
diff --git a/comfy_execution/caching.py b/comfy_execution/caching.py
index 3340e5116..6003bc7bd 100644
--- a/comfy_execution/caching.py
+++ b/comfy_execution/caching.py
@@ -1,11 +1,11 @@
import asyncio
import bisect
import itertools
-import psutil
import time
import torch
from typing import Sequence, Mapping, Dict
from comfy.model_patcher import is_model_patcher_output
+from comfy.system_memory import virtual_memory_available
from comfy_execution.graph import DynamicPrompt
from abc import ABC, abstractmethod
@@ -548,7 +548,7 @@ class RAMPressureCache(LRUCache):
super().set_local(node_id, value)
def ram_release(self, target, free_active=False, min_entry_size=0):
- if psutil.virtual_memory().available >= target:
+ if virtual_memory_available() >= target:
return 0
clean_list = []
@@ -588,7 +588,7 @@ class RAMPressureCache(LRUCache):
bisect.insort(clean_list, (oom_score, self.timestamps[key], key, ram_usage))
freed = 0
- while psutil.virtual_memory().available < target and clean_list:
+ while virtual_memory_available() < target and clean_list:
_, _, key, ram_usage = clean_list.pop()
del self.cache[key]
self.used_generation.pop(key, None)
diff --git a/execution.py b/execution.py
index 0b858969d..62f3d7c52 100644
--- a/execution.py
+++ b/execution.py
@@ -2,7 +2,6 @@ import copy
import heapq
import inspect
import logging
-import psutil
import sys
import threading
import time
@@ -18,6 +17,7 @@ import comfy.memory_management
import comfy.model_management
import comfy.model_patcher
import comfy.model_prefetch
+import comfy.system_memory
import comfy_aimdo.model_vbar
from comfy.internal_logging import detail
@@ -798,7 +798,7 @@ class PromptExecutor:
if self.cache_type == CacheType.RAM_PRESSURE:
ram_release_callback(ram_inactive_headroom)
- ram_shortfall = ram_headroom - psutil.virtual_memory().available
+ ram_shortfall = ram_headroom - comfy.system_memory.virtual_memory_available()
if ram_shortfall > 0:
freed = ram_release_callback(ram_headroom, free_active=True, min_entry_size=RAM_CACHE_LARGE_INTERMEDIATE)
ram_shortfall -= freed
diff --git a/tests-unit/comfy_test/system_memory_test.py b/tests-unit/comfy_test/system_memory_test.py
new file mode 100644
index 000000000..d2d9dfe41
--- /dev/null
+++ b/tests-unit/comfy_test/system_memory_test.py
@@ -0,0 +1,228 @@
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+import comfy.system_memory as system_memory
+
+GIB = 1024 ** 3
+HOST_TOTAL = 128 * GIB
+HOST_AVAILABLE = 100 * GIB
+
+
+def write(path, text):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text)
+
+
+def v2(directory, limit=None, current=None, inactive_file=None):
+ if limit is not None:
+ write(directory / "memory.max", f"{limit}\n")
+ if current is not None:
+ write(directory / "memory.current", f"{current}\n")
+ if inactive_file is not None:
+ write(directory / "memory.stat", f"anon 1\nfile 2\ninactive_file {inactive_file}\nactive_file 3\n")
+
+
+def v1(directory, limit=None, usage=None, inactive_file=None):
+ if limit is not None:
+ write(directory / "memory.limit_in_bytes", f"{limit}\n")
+ if usage is not None:
+ write(directory / "memory.usage_in_bytes", f"{usage}\n")
+ if inactive_file is not None:
+ write(directory / "memory.stat", f"cache 1\ninactive_file 9\ntotal_inactive_file {inactive_file}\n")
+
+
+def cgroup(tmp_path, monkeypatch):
+ v2_root = tmp_path / "sys_fs_cgroup"
+ v1_root = v2_root / "memory"
+ proc = tmp_path / "proc_self_cgroup"
+ v2_root.mkdir()
+ monkeypatch.setattr(system_memory, "CGROUP_V2_ROOT", str(v2_root))
+ monkeypatch.setattr(system_memory, "CGROUP_V1_MEMORY_ROOT", str(v1_root))
+ monkeypatch.setattr(system_memory, "PROC_SELF_CGROUP", str(proc))
+ monkeypatch.setattr(system_memory, "_cgroup_dirs", None)
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setattr(system_memory.psutil, "virtual_memory", lambda: SimpleNamespace(total=HOST_TOTAL, available=HOST_AVAILABLE))
+ return SimpleNamespace(v2_root=v2_root, v1_root=v1_root, proc_path=proc, proc=lambda text: write(proc, text))
+
+
+def assert_host_values():
+ assert system_memory.cgroup_memory_limit() is None
+ assert system_memory.virtual_memory_total() == HOST_TOTAL
+ assert system_memory.virtual_memory_available() == HOST_AVAILABLE
+
+
[email protected]("platform", ["darwin", "win32"])
+def test_non_linux_returns_psutil_values(cgroup, monkeypatch, platform):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=32 * GIB, current=8 * GIB, inactive_file=0)
+ monkeypatch.setattr(sys, "platform", platform)
+ assert_host_values()
+
+
+def test_no_cgroup_files_returns_psutil_values(cgroup):
+ cgroup.proc("0::/\n")
+ assert_host_values()
+
+
+def test_v2_private_namespace_root_limit(cgroup):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=32 * GIB, current=10 * GIB, inactive_file=0)
+ assert system_memory.cgroup_memory_limit() == 32 * GIB
+ assert system_memory.virtual_memory_total() == 32 * GIB
+ assert system_memory.virtual_memory_available() == 22 * GIB
+
+
[email protected]("limit", ["max", "0", "-1", ""])
+def test_v2_unlimited_and_unusable_limit_values_are_ignored(cgroup, limit):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=limit, current=8 * GIB, inactive_file=0)
+ assert_host_values()
+
+
+def test_v2_nested_host_namespace_path(cgroup):
+ cgroup.proc("0::/kubepods.slice/pod1/ctr1\n")
+ v2(cgroup.v2_root, current=100 * GIB, inactive_file=0)
+ v2(cgroup.v2_root / "kubepods.slice" / "pod1" / "ctr1", limit=16 * GIB, current=4 * GIB, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 16 * GIB
+ assert system_memory.virtual_memory_available() == 12 * GIB
+
+
+def test_v2_parent_limit_applies_when_own_cgroup_is_unlimited(cgroup):
+ cgroup.proc("0::/pod/ctr\n")
+ v2(cgroup.v2_root / "pod" / "ctr", limit="max", current=4 * GIB, inactive_file=0)
+ v2(cgroup.v2_root / "pod", limit=16 * GIB, current=6 * GIB, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 16 * GIB
+ assert system_memory.virtual_memory_available() == 10 * GIB
+
+
+def test_v2_leaf_headroom_binds_when_parent_has_more_headroom(cgroup):
+ cgroup.proc("0::/pod/ctr\n")
+ v2(cgroup.v2_root / "pod" / "ctr", limit=16 * GIB, current=4 * GIB, inactive_file=0)
+ v2(cgroup.v2_root / "pod", limit=64 * GIB, current=40 * GIB, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 16 * GIB
+ assert system_memory.virtual_memory_available() == 12 * GIB
+
+
+def test_v2_parent_headroom_binds_when_smaller_than_leaf_headroom(cgroup):
+ cgroup.proc("0::/pod/ctr\n")
+ v2(cgroup.v2_root / "pod" / "ctr", limit=16 * GIB, current=4 * GIB, inactive_file=0)
+ v2(cgroup.v2_root / "pod", limit=32 * GIB, current=31 * GIB, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 16 * GIB
+ assert system_memory.virtual_memory_available() == GIB
+
+
+def test_v2_equal_limits_count_sibling_usage_at_parent(cgroup):
+ cgroup.proc("0::/pod/ctr\n")
+ v2(cgroup.v2_root / "pod" / "ctr", limit=32 * GIB, current=4 * GIB, inactive_file=0)
+ v2(cgroup.v2_root / "pod", limit=32 * GIB, current=20 * GIB, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 32 * GIB
+ assert system_memory.virtual_memory_available() == 12 * GIB
+
+
+def test_v1_limit_and_usage(cgroup):
+ cgroup.proc("4:memory:/docker/abc\n3:cpu,cpuacct:/docker/abc\n")
+ v1(cgroup.v1_root / "docker" / "abc", limit=32 * GIB, usage=10 * GIB, inactive_file=2 * GIB)
+ assert system_memory.cgroup_memory_limit() == 32 * GIB
+ assert system_memory.virtual_memory_total() == 32 * GIB
+ assert system_memory.virtual_memory_available() == 24 * GIB
+
+
+def test_v1_docker_bind_mounted_root(cgroup):
+ cgroup.proc("4:memory:/docker/abc\n")
+ v1(cgroup.v1_root, limit=32 * GIB, usage=10 * GIB, inactive_file=2 * GIB)
+ assert system_memory.virtual_memory_total() == 32 * GIB
+ assert system_memory.virtual_memory_available() == 24 * GIB
+
+
+def test_v1_line_without_memory_controller_is_ignored(cgroup):
+ cgroup.proc("3:cpu,cpuacct:/foo\n")
+ v1(cgroup.v1_root / "foo", limit=8 * GIB, usage=GIB, inactive_file=0)
+ assert_host_values()
+
+
+def test_hybrid_layout_finds_v1_memory_limit(cgroup):
+ cgroup.proc("4:memory:/docker/abc\n0::/\n")
+ v1(cgroup.v1_root / "docker" / "abc", limit=32 * GIB, usage=10 * GIB, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 32 * GIB
+ assert system_memory.virtual_memory_available() == 22 * GIB
+
+
[email protected]("limit", [HOST_TOTAL, 2 * HOST_TOTAL])
+def test_limit_at_or_above_host_total_is_unlimited(cgroup, limit):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=limit, current=8 * GIB, inactive_file=0)
+ assert_host_values()
+
+
+def test_page_cache_is_not_counted_as_used(cgroup):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=96_600_000_000, current=96_600_000_000, inactive_file=49_500_000_000)
+ assert system_memory.virtual_memory_available() == 49_500_000_000
+
+
+def test_available_is_capped_by_host_available(cgroup, monkeypatch):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=64 * GIB, current=0, inactive_file=0)
+ monkeypatch.setattr(system_memory.psutil, "virtual_memory", lambda: SimpleNamespace(total=HOST_TOTAL, available=10 * GIB))
+ assert system_memory.virtual_memory_available() == 10 * GIB
+
+
+def test_available_is_never_negative(cgroup):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=16 * GIB, current=20 * GIB, inactive_file=0)
+ assert system_memory.virtual_memory_available() == 0
+
+
+def test_available_falls_back_to_limit_when_usage_is_unreadable(cgroup):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=32 * GIB)
+ assert system_memory.virtual_memory_total() == 32 * GIB
+ assert system_memory.virtual_memory_available() == 32 * GIB
+
+
[email protected]("stat", [None, "inactive_file abc\n", "anon 1\n"])
+def test_unusable_memory_stat_counts_all_usage(cgroup, stat):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=32 * GIB, current=10 * GIB)
+ if stat is not None:
+ write(cgroup.v2_root / "memory.stat", stat)
+ assert system_memory.virtual_memory_available() == 22 * GIB
+
+
+def test_runtime_limit_change_is_observed(cgroup):
+ cgroup.proc("0::/\n")
+ v2(cgroup.v2_root, limit=64 * GIB, current=0, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 64 * GIB
+ v2(cgroup.v2_root, limit=32 * GIB)
+ assert system_memory.virtual_memory_total() == 32 * GIB
+ assert system_memory.virtual_memory_available() == 32 * GIB
+ v2(cgroup.v2_root, limit="max")
+ assert_host_values()
+
+
+def test_cgroup_path_is_resolved_once(cgroup):
+ cgroup.proc("0::/a\n")
+ v2(cgroup.v2_root / "a", limit=16 * GIB, current=0, inactive_file=0)
+ v2(cgroup.v2_root / "b", limit=8 * GIB, current=0, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 16 * GIB
+ cgroup.proc("0::/b\n")
+ assert system_memory.virtual_memory_total() == 16 * GIB
+
+
[email protected]("proc", [None, b"0::/\xff\n"])
+def test_unusable_proc_self_cgroup_falls_back_to_well_known_roots(cgroup, proc):
+ if proc is not None:
+ cgroup.proc_path.write_bytes(proc)
+ v2(cgroup.v2_root, limit=32 * GIB, current=8 * GIB, inactive_file=0)
+ assert system_memory.virtual_memory_total() == 32 * GIB
+ assert system_memory.virtual_memory_available() == 24 * GIB
+
+
+def test_well_known_roots_are_not_consulted_when_proc_lists_a_cgroup(cgroup):
+ cgroup.proc("0::/a\n")
+ (cgroup.v2_root / "a").mkdir()
+ v1(cgroup.v1_root, limit=8 * GIB, usage=GIB, inactive_file=0)
+ assert_host_values()