The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
fix(assets): don't take the database lock when assets are off; warn when it's held (#16742)
Simon Pinfold2e0316ea50f0
diff --git a/app/assets/manager.py b/app/assets/manager.pyindex 73c351266..0cf812fcd 100644--- a/app/assets/manager.py+++ b/app/assets/manager.py@@ -94,7 +94,6 @@ class NoAssets:def startup(self) -> None:mode.init(self._args)- record_hash_mode_transition_intent()run_startup(enable_assets=False)def shutdown(self) -> None:diff --git a/app/database/db.py b/app/database/db.pyindex 32065ccac..154953913 100644--- a/app/database/db.py+++ b/app/database/db.py@@ -197,6 +197,31 @@ def _acquire_file_lock(db_path):)+def lock_holder_db_path():+ """The database path if another process holds its lock, else None.++ Never waits and never keeps the lock: a free lock is taken and released at once.+ A missing lock file means no holder, so none is created.+ """+ try:+ db_path = get_db_path()+ except ValueError:+ return None+ lock_path = db_path + ".lock"+ if not os.path.exists(lock_path):+ return None+ probe = FileLock(lock_path)+ try:+ probe.acquire(timeout=0)+ probe.release()+ except Timeout:+ return db_path+ except Exception as e:+ # The check is advisory, so it must never stop startup.+ logging.debug(f"Could not check the database lock '{lock_path}': {e}")+ return None++def _is_memory_db(db_url):"""Check if the database URL refers to an in-memory SQLite database."""return db_url in ("sqlite:///:memory:", "sqlite://")diff --git a/main.py b/main.pyindex 92cad0420..62fe2e582 100644--- a/main.py+++ b/main.py@@ -22,7 +22,7 @@ console_log_level = get_console_log_level(args.verbose)file_log_outputs = get_file_log_outputs(args.verbose)setup_logger(log_level=console_log_level, file_outputs=file_log_outputs, use_stdout=args.log_stdout)-from app.database.db import dependencies_available, init_db+from app.database.db import dependencies_available, init_db, lock_holder_db_pathfrom app.assets.lifecycle import cleanup_temp_filesystemfrom app.assets.manager import AssetManager, default_asset_managerimport itertools@@ -469,37 +469,52 @@ def setup_database(asset_manager):if not dependencies_available():return+ if not asset_manager.enabled:+ # Only the asset system uses the database, so leave it, and its lock, to a process that has assets on.+ warn_if_database_in_use()+ asset_manager.startup()+ return+try:init_db()asset_manager.startup()except Exception as e:- if "database is locked" in str(e):+ if "database is locked" in str(e) or "Could not acquire lock on database" in str(e):logging.error("Database is locked. Another ComfyUI process is already using this database.\n""To resolve this, specify a separate database file for this instance:\n"" --database-url sqlite:///path/to/another.db")sys.exit(1)- if "Could not acquire lock on database" in str(e):- logging.error(- "Database is locked. Another ComfyUI process is already using this database.\n"- "To resolve this, specify a separate database file for this instance:\n"- " --database-url sqlite:///path/to/another.db"- )- if args.enable_assets:- sys.exit(1)- return- if args.enable_assets:- logging.error(- f"Failed to initialize database: {e}\n"- "The --enable-assets flag requires a working database connection.\n"- "To resolve this, try one of the following:\n"- " 1. Install the latest requirements: pip install -r requirements.txt\n"- " 2. Specify an alternative database URL: --database-url sqlite:///path/to/your.db\n"- " 3. Use an in-memory database: --database-url sqlite:///:memory:"- )- sys.exit(1)- logging.error(f"Failed to initialize database. Please ensure you have installed the latest requirements. If the error persists, please report this as in future the database will be required: {e}")+ logging.error(+ f"Failed to initialize database: {e}\n"+ "The --enable-assets flag requires a working database connection.\n"+ "To resolve this, try one of the following:\n"+ " 1. Install the latest requirements: pip install -r requirements.txt\n"+ " 2. Specify an alternative database URL: --database-url sqlite:///path/to/your.db\n"+ " 3. Use an in-memory database: --database-url sqlite:///:memory:"+ )+ sys.exit(1)+++def warn_if_database_in_use():+ db_path = lock_holder_db_path()+ if db_path is None:+ return+ app.logger.log_startup_warning(+ f"""+________________________________________________________________________+WARNING WARNING WARNING WARNING WARNING++Another ComfyUI is already using this install's asset database:+ {db_path}+This ComfyUI was started without --enable-assets, so it doesn't need that database and will start anyway.+A future version will refuse to start two ComfyUIs on the same asset database.+To run both, give this one its own:+ --database-url sqlite:///path/to/another.db+________________________________________________________________________+""".strip()+ )def start_comfyui(asyncio_loop=None):diff --git a/tests-unit/app_test/test_db_assets_off.py b/tests-unit/app_test/test_db_assets_off.pynew file mode 100644index 000000000..8a08c8177--- /dev/null+++ b/tests-unit/app_test/test_db_assets_off.py@@ -0,0 +1,142 @@+import os++import pytest+import torch+from filelock import FileLock, Timeout++import app.logger+from app.database import db as db_module+from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import main # noqa: E402+++def db_path(tmp_path, monkeypatch):+ path = str(tmp_path / "comfyui.db")+ monkeypatch.setattr(db_module.args, "database_url", f"sqlite:///{path}")+ return path+++def startup_warnings(monkeypatch):+ warnings = []+ monkeypatch.setattr(app.logger, "STARTUP_WARNINGS", warnings)+ return warnings+++class _AssetsOff:+ enabled = False++ def __init__(self):+ self.started = False++ def startup(self):+ self.started = True+++def test_probe_without_a_lock_file_creates_none(db_path):+ assert db_module.lock_holder_db_path() is None+ assert not os.path.exists(db_path + ".lock")+++def test_probe_reports_a_held_lock(db_path):+ holder = FileLock(db_path + ".lock")+ holder.acquire(timeout=0)+ try:+ assert db_module.lock_holder_db_path() == db_path+ finally:+ holder.release()+++def test_probe_does_not_keep_a_free_lock(db_path, monkeypatch):+ open(db_path + ".lock", "a").close()+ probes = []++ class _KeptAlive(FileLock):+ # Holding a reference stops garbage collection from releasing a lock the probe kept.+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ probes.append(self)++ monkeypatch.setattr(db_module, "FileLock", _KeptAlive)++ assert db_module.lock_holder_db_path() is None+ assert len(probes) == 1++ contender = FileLock(db_path + ".lock")+ try:+ contender.acquire(timeout=0)+ except Timeout:+ pytest.fail("the probe kept the lock it was only meant to test")+ contender.release()+++def test_probe_skips_non_sqlite_databases(monkeypatch):+ monkeypatch.setattr(db_module.args, "database_url", "postgresql://localhost/comfy")++ assert db_module.lock_holder_db_path() is None+++def test_probe_treats_a_failed_check_as_free(db_path, monkeypatch):+ open(db_path + ".lock", "a").close()++ def _unexpected(self, *args, **kwargs):+ raise NotImplementedError("no locking on this filesystem")++ monkeypatch.setattr(FileLock, "acquire", _unexpected)++ assert db_module.lock_holder_db_path() is None+++def test_assets_off_leaves_the_database_alone(db_path, startup_warnings, monkeypatch):+ monkeypatch.setattr(main, "dependencies_available", lambda: True)++ def _init_db():+ pytest.fail("init_db ran with assets off")++ monkeypatch.setattr(main, "init_db", _init_db)+ asset_manager = _AssetsOff()++ main.setup_database(asset_manager)++ assert asset_manager.started+ assert not os.path.exists(db_path)+ assert not os.path.exists(db_path + ".lock")+ assert startup_warnings == []+++def test_assets_off_warns_and_continues_when_another_process_holds_the_lock(+ db_path, startup_warnings, monkeypatch+):+ monkeypatch.setattr(main, "dependencies_available", lambda: True)+ monkeypatch.setattr(main, "init_db", lambda: pytest.fail("init_db ran with assets off"))+ asset_manager = _AssetsOff()+ holder = FileLock(db_path + ".lock")+ holder.acquire(timeout=0)+ try:+ main.setup_database(asset_manager)+ finally:+ holder.release()++ assert asset_manager.started+ assert len(startup_warnings) == 1+ warning = startup_warnings[0]+ assert "Another ComfyUI is already using this install's asset database" in warning+ assert db_path in warning+ assert "This ComfyUI was started without --enable-assets, so it doesn't need that database and will start anyway" in warning+ assert "A future version will refuse to start two ComfyUIs on the same asset database" in warning+++def test_probe_treats_a_failed_release_as_free(db_path, monkeypatch):+ open(db_path + ".lock", "a").close()++ def _unexpected(self, *args, **kwargs):+ raise OSError("unlock failed")++ monkeypatch.setattr(FileLock, "release", _unexpected)++ assert db_module.lock_holder_db_path() is Nonediff --git a/tests-unit/app_test/test_db_init_locking.py b/tests-unit/app_test/test_db_init_locking.pyindex c31065432..b4aa9bca3 100644--- a/tests-unit/app_test/test_db_init_locking.py+++ b/tests-unit/app_test/test_db_init_locking.py@@ -76,9 +76,8 @@ def test_failed_init_releases_the_lock(stale_db, monkeypatch):contender.acquire(timeout=0)except Timeout:pytest.fail(- "a failed init stranded the lock; setup_database logs and CONTINUES when assets are "- "disabled, so this process would block every other instance for its whole lifetime "- "over a database it never opened"+ "a failed init stranded the lock, so this process would block every other "+ "instance for its whole lifetime over a database it never opened")contender.release()@@ -135,25 +134,11 @@ def test_legacy_database_copy_runs_under_file_lock(tmp_path, monkeypatch):assert copied == [(str(legacy_db) + ".bak", str(target_db))]-def test_setup_database_routes_file_lock_to_lock_guidance(monkeypatch, caplog):- monkeypatch.setattr(main, "dependencies_available", lambda: True)-- def _raise_file_lock():- raise RuntimeError(- "Could not acquire lock on database 'x.db'. "- "Another ComfyUI process may already be using it. "- "Use --database-url to specify a separate database file."- )-- monkeypatch.setattr(main, "init_db", _raise_file_lock)- monkeypatch.setattr(main.args, "enable_assets", False)-- with caplog.at_level(logging.ERROR):- result = main.setup_database(None)+class _AssetsOn:+ enabled = True- assert result is None- assert "Database is locked. Another ComfyUI process is already using this database." in caplog.text- assert "Failed to initialize database." not in caplog.text+ def startup(self):+ passdef test_setup_database_exits_for_file_lock_when_assets_are_enabled(monkeypatch, caplog):@@ -167,32 +152,46 @@ def test_setup_database_exits_for_file_lock_when_assets_are_enabled(monkeypatch,)monkeypatch.setattr(main, "init_db", _raise_file_lock)- monkeypatch.setattr(main.args, "enable_assets", True)with caplog.at_level(logging.ERROR), pytest.raises(SystemExit) as error:- main.setup_database(None)+ main.setup_database(_AssetsOn())assert error.value.code == 1assert "Database is locked. Another ComfyUI process is already using this database." in caplog.textassert "The --enable-assets flag requires a working database connection." not in caplog.text-def test_setup_database_exits_for_driver_lock_when_assets_are_disabled(monkeypatch, caplog):+def test_setup_database_exits_for_driver_lock_when_assets_are_enabled(monkeypatch, caplog):monkeypatch.setattr(main, "dependencies_available", lambda: True)def _raise_driver_lock():raise Exception("sqlite3.OperationalError: database is locked")monkeypatch.setattr(main, "init_db", _raise_driver_lock)- monkeypatch.setattr(main.args, "enable_assets", False)with caplog.at_level(logging.ERROR), pytest.raises(SystemExit) as error:- main.setup_database(None)+ main.setup_database(_AssetsOn())assert error.value.code == 1assert "Database is locked. Another ComfyUI process is already using this database." in caplog.text+def test_setup_database_exits_for_other_failures_when_assets_are_enabled(monkeypatch, caplog):+ monkeypatch.setattr(main, "dependencies_available", lambda: True)++ def _raise_other():+ raise RuntimeError("alembic config exploded")++ monkeypatch.setattr(main, "init_db", _raise_other)++ with caplog.at_level(logging.ERROR), pytest.raises(SystemExit) as error:+ main.setup_database(_AssetsOn())++ assert error.value.code == 1+ assert "The --enable-assets flag requires a working database connection." in caplog.text+ assert "Database is locked." not in caplog.text++def test_failed_restore_does_not_mask_the_upgrade_error(stale_db, monkeypatch, caplog):real_backup = db_module._backup_databasediff --git a/tests-unit/app_test/test_db_lock_processes.py b/tests-unit/app_test/test_db_lock_processes.pynew file mode 100644index 000000000..41cb0a794--- /dev/null+++ b/tests-unit/app_test/test_db_lock_processes.py@@ -0,0 +1,133 @@+"""Real ComfyUI processes sharing one install's database: an assets-off instance must+not block an assets-on one, and must warn when another process holds the database."""++import socket+import subprocess+import sys+import time+import urllib.request+from pathlib import Path++import pytest++REPO_ROOT = Path(__file__).resolve().parents[2]+LOCK_HELD = "Database is locked. Another ComfyUI process is already using this database."+IN_USE = "Another ComfyUI is already using this install's asset database"++HOLD_SCRIPT = (+ "import sys; "+ "from filelock import FileLock; "+ "lock = FileLock(sys.argv[1]); lock.acquire(timeout=0); "+ "print('held', flush=True); "+ "sys.stdin.readline()"+)+++def _comfy_args(base: Path, *flags: str) -> list[str]:+ return [+ sys.executable,+ "main.py",+ "--cpu",+ "--disable-all-custom-nodes",+ "--disable-partner-nodes",+ f"--base-directory={base}",+ f"--front-end-root={base}",+ f"--database-url=sqlite:///{base / 'comfyui.db'}",+ *flags,+ ]+++def _quick_start(base: Path, *flags: str) -> subprocess.CompletedProcess:+ return subprocess.run(+ _comfy_args(base, "--quick-test-for-ci", *flags),+ cwd=REPO_ROOT,+ capture_output=True,+ text=True,+ timeout=120,+ )+++def _free_port() -> int:+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _wait_until_serving(proc: subprocess.Popen, port: int, timeout: float = 120.0) -> None:+ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ if proc.poll() is not None:+ raise AssertionError(f"ComfyUI exited with code {proc.returncode} before serving")+ try:+ with urllib.request.urlopen(f"http://127.0.0.1:{port}/system_stats", timeout=2) as response:+ if response.status == 200:+ return+ except OSError:+ pass+ time.sleep(0.2)+ raise AssertionError("ComfyUI did not start serving")+++def held_lock(tmp_path):+ holder = subprocess.Popen(+ [sys.executable, "-c", HOLD_SCRIPT, str(tmp_path / "comfyui.db.lock")],+ cwd=REPO_ROOT,+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ text=True,+ )+ try:+ assert holder.stdout.readline().strip() == "held"+ yield+ finally:+ holder.kill()+ holder.wait()+++def test_running_assets_off_instance_does_not_block_an_assets_on_start(tmp_path):+ # A previous assets-on run leaves the database and its lock file behind, as on a real install.+ first = _quick_start(tmp_path, "--enable-assets")+ assert first.returncode == 0, first.stderr++ port = _free_port()+ server_log = tmp_path / "server.log"+ with open(server_log, "w") as log:+ server = subprocess.Popen(+ _comfy_args(tmp_path, "--listen", "127.0.0.1", "--port", str(port)),+ cwd=REPO_ROOT,+ stdout=log,+ stderr=subprocess.STDOUT,+ )+ try:+ try:+ _wait_until_serving(server, port)+ except AssertionError as e:+ raise AssertionError(f"{e}\n{server_log.read_text()[-4000:]}") from None++ second = _quick_start(tmp_path, "--enable-assets")++ assert server.poll() is None+ finally:+ server.terminate()+ server.wait(timeout=30)++ assert second.returncode == 0, second.stderr+ assert LOCK_HELD not in second.stderr+++def test_assets_off_start_warns_and_continues_when_the_database_is_held(tmp_path, held_lock):+ result = _quick_start(tmp_path)++ assert result.returncode == 0, result.stderr+ assert IN_USE in result.stderr+ assert str(tmp_path / "comfyui.db") in result.stderr+ assert "A future version will refuse to start two ComfyUIs on the same asset database" in result.stderr+ assert "Traceback" not in result.stderr+++def test_assets_on_start_still_fails_when_the_database_is_held(tmp_path, held_lock):+ result = _quick_start(tmp_path, "--enable-assets")++ assert result.returncode == 1, result.stderr+ assert LOCK_HELD in result.stderrdiff --git a/tests-unit/assets_test/services/test_noassets_parity.py b/tests-unit/assets_test/services/test_noassets_parity.pyindex 121582f95..e1ff69ac4 100644--- a/tests-unit/assets_test/services/test_noassets_parity.py+++ b/tests-unit/assets_test/services/test_noassets_parity.py@@ -49,7 +49,7 @@ async def test_noassets_register_routes_returns_service_disabled_and_disables_seassert asset_seeder.is_disabled()-def test_noassets_startup_applies_hash_mode_persists_state_and_cleans_temp_dir(+def test_noassets_startup_applies_hash_mode_leaves_database_alone_and_cleans_temp_dir(mock_create_session: Callable[[], AbstractContextManager[Session]],session: Session,tmp_path: Path,@@ -63,7 +63,7 @@ def test_noassets_startup_applies_hash_mode_persists_state_and_cleans_temp_dir(_no_assets(hashing=True).startup()assert hashing_enabled() is True- assert read_stored_mode(session) == "on"+ assert read_stored_mode(session) is Noneassert not temp_file.exists()