The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
fix(memory): respect the container cgroup memory limit instead of host RAM (#15927)
Alexander Piskun77739723a36e
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex ff963eb8e..ffc4f129d 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -30,6 +30,7 @@ import gcimport osfrom contextlib import contextmanager, nullcontextimport comfy.memory_management+import comfy.system_memoryimport comfy.utilsimport comfy.quant_opsimport comfy_aimdo.host_buffer@@ -318,7 +319,7 @@ def get_total_memory(dev=None, torch_total_too=False):dev = get_torch_device()if hasattr(dev, 'type') and (dev.type == 'cpu' or dev.type == 'mps'):- mem_total = psutil.virtual_memory().total+ mem_total = comfy.system_memory.virtual_memory_total()mem_total_torch = mem_totalelse:if directml_enabled:@@ -361,8 +362,11 @@ def mac_version():return Nonetotal_vram = get_total_memory(get_torch_device()) / (1024 * 1024)-total_ram = psutil.virtual_memory().total / (1024 * 1024)+total_ram = comfy.system_memory.virtual_memory_total() / (1024 * 1024)logging.info("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram))+cgroup_ram_limit = comfy.system_memory.cgroup_memory_limit()+if cgroup_ram_limit is not None:+ logging.info("RAM limited by cgroup to {:0.0f} MB (host has {:0.0f} MB)".format(cgroup_ram_limit / (1024 * 1024), psutil.virtual_memory().total / (1024 * 1024)))try:logging.info("pytorch version: {}".format(torch_version))@@ -703,7 +707,7 @@ def should_free_pins_for_ram_pressure(shortfall):return Falseif not WINDOWS:return True- if psutil.virtual_memory().available < WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE:+ if comfy.system_memory.virtual_memory_available() < WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE:return Truetry:return psutil.swap_memory().percent >= WINDOWS_PIN_EVICTION_SWAP_PERCENT@@ -717,7 +721,7 @@ def ensure_pin_budget(size, evict_active=False, loaded=False):if args.fast_disk:shortfall = TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORYelse:- shortfall = size + max(comfy.memory_management.RAM_CACHE_HEADROOM / 2, 2048 * 1024 ** 2) - psutil.virtual_memory().available+ shortfall = size + max(comfy.memory_management.RAM_CACHE_HEADROOM / 2, 2048 * 1024 ** 2) - comfy.system_memory.virtual_memory_available()if shortfall <= 0:return True@@ -1584,7 +1588,8 @@ if not args.disable_pinned_memory:if WINDOWS:MAX_PINNED_MEMORY = ram * 0.40 # Windows limit is apparently 50%else:- MAX_PINNED_MEMORY = max(ram * 0.40, min(ram * 0.90, ram - 4 * 1024 ** 3, ram + get_disk_swap_total() - 16 * 1024 ** 3))+ swap = 0 if comfy.system_memory.cgroup_memory_limit() is not None else get_disk_swap_total()+ MAX_PINNED_MEMORY = max(ram * 0.40, min(ram * 0.90, ram - 4 * 1024 ** 3, ram + swap - 16 * 1024 ** 3))logging.info("Enabled pinned memory {}".format(MAX_PINNED_MEMORY // (1024 * 1024)))PINNING_ALLOWED_TYPES = set(["Tensor", "Parameter", "QuantizedTensor"])@@ -1751,7 +1756,7 @@ def get_free_memory(dev=None, torch_free_too=False):dev = get_torch_device()if hasattr(dev, 'type') and (dev.type == 'cpu' or dev.type == 'mps'):- mem_free_total = psutil.virtual_memory().available+ mem_free_total = comfy.system_memory.virtual_memory_available()mem_free_torch = mem_free_totalelse:if directml_enabled:diff --git a/comfy/system_memory.py b/comfy/system_memory.pynew file mode 100644index 000000000..63de575f8--- /dev/null+++ b/comfy/system_memory.py@@ -0,0 +1,128 @@+import os+import sys++import psutil++CGROUP_V2_ROOT = "/sys/fs/cgroup"+CGROUP_V1_MEMORY_ROOT = "/sys/fs/cgroup/memory"+PROC_SELF_CGROUP = "/proc/self/cgroup"++_cgroup_dirs = None+++def _read_text(path):+ try:+ with open(path, encoding="utf-8") as f:+ return f.read().strip()+ except (OSError, ValueError):+ return None+++def _read_int(path):+ raw = _read_text(path)+ if raw is None:+ return None+ try:+ return int(raw)+ except ValueError:+ return None+++def _read_stat(path, key):+ raw = _read_text(path)+ if raw is None:+ return None+ for line in raw.splitlines():+ parts = line.split()+ if len(parts) == 2 and parts[0] == key:+ try:+ return int(parts[1])+ except ValueError:+ return None+ return None+++def _lineage(root, path):+ dirs = [root]+ for part in path.split("/"):+ if part:+ dirs.append(os.path.join(dirs[-1], part))+ return dirs[::-1]+++def _own_cgroup_dirs():+ raw = _read_text(PROC_SELF_CGROUP)+ if raw is None:+ return []+ dirs = []+ for line in raw.splitlines():+ parts = line.split(":", 2)+ if len(parts) != 3:+ continue+ controllers, path = parts[1], parts[2]+ if controllers == "":+ root = CGROUP_V2_ROOT+ elif "memory" in controllers.split(","):+ root = CGROUP_V1_MEMORY_ROOT+ else:+ continue+ for directory in _lineage(root, path):+ if directory not in dirs:+ dirs.append(directory)+ return dirs+++def _cgroup_directories():+ global _cgroup_dirs+ if _cgroup_dirs is None:+ _cgroup_dirs = _own_cgroup_dirs() or [CGROUP_V2_ROOT, CGROUP_V1_MEMORY_ROOT]+ return _cgroup_dirs+++def _limit_in(directory):+ for name in ("memory.max", "memory.limit_in_bytes"):+ value = _read_int(os.path.join(directory, name))+ if value is not None and value > 0:+ return value+ return None+++def _working_set_in(directory):+ usage = _read_int(os.path.join(directory, "memory.current"))+ key = "inactive_file"+ if usage is None:+ usage = _read_int(os.path.join(directory, "memory.usage_in_bytes"))+ key = "total_inactive_file"+ if usage is None:+ return None+ inactive_file = _read_stat(os.path.join(directory, "memory.stat"), key) or 0+ return max(0, usage - inactive_file)+++def _limited(host_total):+ if not sys.platform.startswith("linux"):+ return []+ limited = []+ for directory in _cgroup_directories():+ limit = _limit_in(directory)+ if limit is not None and limit < host_total:+ limited.append((limit, directory))+ return limited+++def cgroup_memory_limit():+ return min((limit for limit, _ in _limited(psutil.virtual_memory().total)), default=None)+++def virtual_memory_total():+ host = psutil.virtual_memory()+ return min((limit for limit, _ in _limited(host.total)), default=host.total)+++def virtual_memory_available():+ host = psutil.virtual_memory()+ available = host.available+ for limit, directory in _limited(host.total):+ used = _working_set_in(directory)+ available = min(available, limit if used is None else limit - used)+ return max(0, available)diff --git a/comfy_execution/caching.py b/comfy_execution/caching.pyindex 3340e5116..6003bc7bd 100644--- a/comfy_execution/caching.py+++ b/comfy_execution/caching.py@@ -1,11 +1,11 @@import asyncioimport bisectimport itertools-import psutilimport timeimport torchfrom typing import Sequence, Mapping, Dictfrom comfy.model_patcher import is_model_patcher_output+from comfy.system_memory import virtual_memory_availablefrom comfy_execution.graph import DynamicPromptfrom abc import ABC, abstractmethod@@ -548,7 +548,7 @@ class RAMPressureCache(LRUCache):super().set_local(node_id, value)def ram_release(self, target, free_active=False, min_entry_size=0):- if psutil.virtual_memory().available >= target:+ if virtual_memory_available() >= target:return 0clean_list = []@@ -588,7 +588,7 @@ class RAMPressureCache(LRUCache):bisect.insort(clean_list, (oom_score, self.timestamps[key], key, ram_usage))freed = 0- while psutil.virtual_memory().available < target and clean_list:+ while virtual_memory_available() < target and clean_list:_, _, key, ram_usage = clean_list.pop()del self.cache[key]self.used_generation.pop(key, None)diff --git a/execution.py b/execution.pyindex 0b858969d..62f3d7c52 100644--- a/execution.py+++ b/execution.py@@ -2,7 +2,6 @@ import copyimport heapqimport inspectimport logging-import psutilimport sysimport threadingimport time@@ -18,6 +17,7 @@ import comfy.memory_managementimport comfy.model_managementimport comfy.model_patcherimport comfy.model_prefetch+import comfy.system_memoryimport comfy_aimdo.model_vbarfrom comfy.internal_logging import detail@@ -798,7 +798,7 @@ class PromptExecutor:if self.cache_type == CacheType.RAM_PRESSURE:ram_release_callback(ram_inactive_headroom)- ram_shortfall = ram_headroom - psutil.virtual_memory().available+ ram_shortfall = ram_headroom - comfy.system_memory.virtual_memory_available()if ram_shortfall > 0:freed = ram_release_callback(ram_headroom, free_active=True, min_entry_size=RAM_CACHE_LARGE_INTERMEDIATE)ram_shortfall -= freeddiff --git a/tests-unit/comfy_test/system_memory_test.py b/tests-unit/comfy_test/system_memory_test.pynew file mode 100644index 000000000..d2d9dfe41--- /dev/null+++ b/tests-unit/comfy_test/system_memory_test.py@@ -0,0 +1,228 @@+import sys+from types import SimpleNamespace++import pytest++import comfy.system_memory as system_memory++GIB = 1024 ** 3+HOST_TOTAL = 128 * GIB+HOST_AVAILABLE = 100 * GIB+++def write(path, text):+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(text)+++def v2(directory, limit=None, current=None, inactive_file=None):+ if limit is not None:+ write(directory / "memory.max", f"{limit}\n")+ if current is not None:+ write(directory / "memory.current", f"{current}\n")+ if inactive_file is not None:+ write(directory / "memory.stat", f"anon 1\nfile 2\ninactive_file {inactive_file}\nactive_file 3\n")+++def v1(directory, limit=None, usage=None, inactive_file=None):+ if limit is not None:+ write(directory / "memory.limit_in_bytes", f"{limit}\n")+ if usage is not None:+ write(directory / "memory.usage_in_bytes", f"{usage}\n")+ if inactive_file is not None:+ write(directory / "memory.stat", f"cache 1\ninactive_file 9\ntotal_inactive_file {inactive_file}\n")+++def cgroup(tmp_path, monkeypatch):+ v2_root = tmp_path / "sys_fs_cgroup"+ v1_root = v2_root / "memory"+ proc = tmp_path / "proc_self_cgroup"+ v2_root.mkdir()+ monkeypatch.setattr(system_memory, "CGROUP_V2_ROOT", str(v2_root))+ monkeypatch.setattr(system_memory, "CGROUP_V1_MEMORY_ROOT", str(v1_root))+ monkeypatch.setattr(system_memory, "PROC_SELF_CGROUP", str(proc))+ monkeypatch.setattr(system_memory, "_cgroup_dirs", None)+ monkeypatch.setattr(sys, "platform", "linux")+ monkeypatch.setattr(system_memory.psutil, "virtual_memory", lambda: SimpleNamespace(total=HOST_TOTAL, available=HOST_AVAILABLE))+ return SimpleNamespace(v2_root=v2_root, v1_root=v1_root, proc_path=proc, proc=lambda text: write(proc, text))+++def assert_host_values():+ assert system_memory.cgroup_memory_limit() is None+ assert system_memory.virtual_memory_total() == HOST_TOTAL+ assert system_memory.virtual_memory_available() == HOST_AVAILABLE++[email protected]("platform", ["darwin", "win32"])+def test_non_linux_returns_psutil_values(cgroup, monkeypatch, platform):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=32 * GIB, current=8 * GIB, inactive_file=0)+ monkeypatch.setattr(sys, "platform", platform)+ assert_host_values()+++def test_no_cgroup_files_returns_psutil_values(cgroup):+ cgroup.proc("0::/\n")+ assert_host_values()+++def test_v2_private_namespace_root_limit(cgroup):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=32 * GIB, current=10 * GIB, inactive_file=0)+ assert system_memory.cgroup_memory_limit() == 32 * GIB+ assert system_memory.virtual_memory_total() == 32 * GIB+ assert system_memory.virtual_memory_available() == 22 * GIB++[email protected]("limit", ["max", "0", "-1", ""])+def test_v2_unlimited_and_unusable_limit_values_are_ignored(cgroup, limit):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=limit, current=8 * GIB, inactive_file=0)+ assert_host_values()+++def test_v2_nested_host_namespace_path(cgroup):+ cgroup.proc("0::/kubepods.slice/pod1/ctr1\n")+ v2(cgroup.v2_root, current=100 * GIB, inactive_file=0)+ v2(cgroup.v2_root / "kubepods.slice" / "pod1" / "ctr1", limit=16 * GIB, current=4 * GIB, inactive_file=0)+ assert system_memory.virtual_memory_total() == 16 * GIB+ assert system_memory.virtual_memory_available() == 12 * GIB+++def test_v2_parent_limit_applies_when_own_cgroup_is_unlimited(cgroup):+ cgroup.proc("0::/pod/ctr\n")+ v2(cgroup.v2_root / "pod" / "ctr", limit="max", current=4 * GIB, inactive_file=0)+ v2(cgroup.v2_root / "pod", limit=16 * GIB, current=6 * GIB, inactive_file=0)+ assert system_memory.virtual_memory_total() == 16 * GIB+ assert system_memory.virtual_memory_available() == 10 * GIB+++def test_v2_leaf_headroom_binds_when_parent_has_more_headroom(cgroup):+ cgroup.proc("0::/pod/ctr\n")+ v2(cgroup.v2_root / "pod" / "ctr", limit=16 * GIB, current=4 * GIB, inactive_file=0)+ v2(cgroup.v2_root / "pod", limit=64 * GIB, current=40 * GIB, inactive_file=0)+ assert system_memory.virtual_memory_total() == 16 * GIB+ assert system_memory.virtual_memory_available() == 12 * GIB+++def test_v2_parent_headroom_binds_when_smaller_than_leaf_headroom(cgroup):+ cgroup.proc("0::/pod/ctr\n")+ v2(cgroup.v2_root / "pod" / "ctr", limit=16 * GIB, current=4 * GIB, inactive_file=0)+ v2(cgroup.v2_root / "pod", limit=32 * GIB, current=31 * GIB, inactive_file=0)+ assert system_memory.virtual_memory_total() == 16 * GIB+ assert system_memory.virtual_memory_available() == GIB+++def test_v2_equal_limits_count_sibling_usage_at_parent(cgroup):+ cgroup.proc("0::/pod/ctr\n")+ v2(cgroup.v2_root / "pod" / "ctr", limit=32 * GIB, current=4 * GIB, inactive_file=0)+ v2(cgroup.v2_root / "pod", limit=32 * GIB, current=20 * GIB, inactive_file=0)+ assert system_memory.virtual_memory_total() == 32 * GIB+ assert system_memory.virtual_memory_available() == 12 * GIB+++def test_v1_limit_and_usage(cgroup):+ cgroup.proc("4:memory:/docker/abc\n3:cpu,cpuacct:/docker/abc\n")+ v1(cgroup.v1_root / "docker" / "abc", limit=32 * GIB, usage=10 * GIB, inactive_file=2 * GIB)+ assert system_memory.cgroup_memory_limit() == 32 * GIB+ assert system_memory.virtual_memory_total() == 32 * GIB+ assert system_memory.virtual_memory_available() == 24 * GIB+++def test_v1_docker_bind_mounted_root(cgroup):+ cgroup.proc("4:memory:/docker/abc\n")+ v1(cgroup.v1_root, limit=32 * GIB, usage=10 * GIB, inactive_file=2 * GIB)+ assert system_memory.virtual_memory_total() == 32 * GIB+ assert system_memory.virtual_memory_available() == 24 * GIB+++def test_v1_line_without_memory_controller_is_ignored(cgroup):+ cgroup.proc("3:cpu,cpuacct:/foo\n")+ v1(cgroup.v1_root / "foo", limit=8 * GIB, usage=GIB, inactive_file=0)+ assert_host_values()+++def test_hybrid_layout_finds_v1_memory_limit(cgroup):+ cgroup.proc("4:memory:/docker/abc\n0::/\n")+ v1(cgroup.v1_root / "docker" / "abc", limit=32 * GIB, usage=10 * GIB, inactive_file=0)+ assert system_memory.virtual_memory_total() == 32 * GIB+ assert system_memory.virtual_memory_available() == 22 * GIB++[email protected]("limit", [HOST_TOTAL, 2 * HOST_TOTAL])+def test_limit_at_or_above_host_total_is_unlimited(cgroup, limit):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=limit, current=8 * GIB, inactive_file=0)+ assert_host_values()+++def test_page_cache_is_not_counted_as_used(cgroup):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=96_600_000_000, current=96_600_000_000, inactive_file=49_500_000_000)+ assert system_memory.virtual_memory_available() == 49_500_000_000+++def test_available_is_capped_by_host_available(cgroup, monkeypatch):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=64 * GIB, current=0, inactive_file=0)+ monkeypatch.setattr(system_memory.psutil, "virtual_memory", lambda: SimpleNamespace(total=HOST_TOTAL, available=10 * GIB))+ assert system_memory.virtual_memory_available() == 10 * GIB+++def test_available_is_never_negative(cgroup):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=16 * GIB, current=20 * GIB, inactive_file=0)+ assert system_memory.virtual_memory_available() == 0+++def test_available_falls_back_to_limit_when_usage_is_unreadable(cgroup):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=32 * GIB)+ assert system_memory.virtual_memory_total() == 32 * GIB+ assert system_memory.virtual_memory_available() == 32 * GIB++[email protected]("stat", [None, "inactive_file abc\n", "anon 1\n"])+def test_unusable_memory_stat_counts_all_usage(cgroup, stat):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=32 * GIB, current=10 * GIB)+ if stat is not None:+ write(cgroup.v2_root / "memory.stat", stat)+ assert system_memory.virtual_memory_available() == 22 * GIB+++def test_runtime_limit_change_is_observed(cgroup):+ cgroup.proc("0::/\n")+ v2(cgroup.v2_root, limit=64 * GIB, current=0, inactive_file=0)+ assert system_memory.virtual_memory_total() == 64 * GIB+ v2(cgroup.v2_root, limit=32 * GIB)+ assert system_memory.virtual_memory_total() == 32 * GIB+ assert system_memory.virtual_memory_available() == 32 * GIB+ v2(cgroup.v2_root, limit="max")+ assert_host_values()+++def test_cgroup_path_is_resolved_once(cgroup):+ cgroup.proc("0::/a\n")+ v2(cgroup.v2_root / "a", limit=16 * GIB, current=0, inactive_file=0)+ v2(cgroup.v2_root / "b", limit=8 * GIB, current=0, inactive_file=0)+ assert system_memory.virtual_memory_total() == 16 * GIB+ cgroup.proc("0::/b\n")+ assert system_memory.virtual_memory_total() == 16 * GIB++[email protected]("proc", [None, b"0::/\xff\n"])+def test_unusable_proc_self_cgroup_falls_back_to_well_known_roots(cgroup, proc):+ if proc is not None:+ cgroup.proc_path.write_bytes(proc)+ v2(cgroup.v2_root, limit=32 * GIB, current=8 * GIB, inactive_file=0)+ assert system_memory.virtual_memory_total() == 32 * GIB+ assert system_memory.virtual_memory_available() == 24 * GIB+++def test_well_known_roots_are_not_consulted_when_proc_lists_a_cgroup(cgroup):+ cgroup.proc("0::/a\n")+ (cgroup.v2_root / "a").mkdir()+ v1(cgroup.v1_root, limit=8 * GIB, usage=GIB, inactive_file=0)+ assert_host_values()