Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

fix(assets): don't take the database lock when assets are off; warn when it's held (#16742)

Simon Pinfold2e0316ea50f0
diff --git a/app/assets/manager.py b/app/assets/manager.py
index 73c351266..0cf812fcd 100644
--- a/app/assets/manager.py
+++ b/app/assets/manager.py
@@ -94,7 +94,6 @@ class NoAssets:
def startup(self) -> None:
mode.init(self._args)
- record_hash_mode_transition_intent()
run_startup(enable_assets=False)
def shutdown(self) -> None:
diff --git a/app/database/db.py b/app/database/db.py
index 32065ccac..154953913 100644
--- a/app/database/db.py
+++ b/app/database/db.py
@@ -197,6 +197,31 @@ def _acquire_file_lock(db_path):
)
+def lock_holder_db_path():
+ """The database path if another process holds its lock, else None.
+
+ Never waits and never keeps the lock: a free lock is taken and released at once.
+ A missing lock file means no holder, so none is created.
+ """
+ try:
+ db_path = get_db_path()
+ except ValueError:
+ return None
+ lock_path = db_path + ".lock"
+ if not os.path.exists(lock_path):
+ return None
+ probe = FileLock(lock_path)
+ try:
+ probe.acquire(timeout=0)
+ probe.release()
+ except Timeout:
+ return db_path
+ except Exception as e:
+ # The check is advisory, so it must never stop startup.
+ logging.debug(f"Could not check the database lock '{lock_path}': {e}")
+ return None
+
+
def _is_memory_db(db_url):
"""Check if the database URL refers to an in-memory SQLite database."""
return db_url in ("sqlite:///:memory:", "sqlite://")
diff --git a/main.py b/main.py
index 92cad0420..62fe2e582 100644
--- a/main.py
+++ b/main.py
@@ -22,7 +22,7 @@ console_log_level = get_console_log_level(args.verbose)
file_log_outputs = get_file_log_outputs(args.verbose)
setup_logger(log_level=console_log_level, file_outputs=file_log_outputs, use_stdout=args.log_stdout)
-from app.database.db import dependencies_available, init_db
+from app.database.db import dependencies_available, init_db, lock_holder_db_path
from app.assets.lifecycle import cleanup_temp_filesystem
from app.assets.manager import AssetManager, default_asset_manager
import itertools
@@ -469,37 +469,52 @@ def setup_database(asset_manager):
if not dependencies_available():
return
+ if not asset_manager.enabled:
+ # Only the asset system uses the database, so leave it, and its lock, to a process that has assets on.
+ warn_if_database_in_use()
+ asset_manager.startup()
+ return
+
try:
init_db()
asset_manager.startup()
except Exception as e:
- if "database is locked" in str(e):
+ if "database is locked" in str(e) or "Could not acquire lock on database" in str(e):
logging.error(
"Database is locked. Another ComfyUI process is already using this database.\n"
"To resolve this, specify a separate database file for this instance:\n"
" --database-url sqlite:///path/to/another.db"
)
sys.exit(1)
- if "Could not acquire lock on database" in str(e):
- logging.error(
- "Database is locked. Another ComfyUI process is already using this database.\n"
- "To resolve this, specify a separate database file for this instance:\n"
- " --database-url sqlite:///path/to/another.db"
- )
- if args.enable_assets:
- sys.exit(1)
- return
- if args.enable_assets:
- logging.error(
- f"Failed to initialize database: {e}\n"
- "The --enable-assets flag requires a working database connection.\n"
- "To resolve this, try one of the following:\n"
- " 1. Install the latest requirements: pip install -r requirements.txt\n"
- " 2. Specify an alternative database URL: --database-url sqlite:///path/to/your.db\n"
- " 3. Use an in-memory database: --database-url sqlite:///:memory:"
- )
- sys.exit(1)
- logging.error(f"Failed to initialize database. Please ensure you have installed the latest requirements. If the error persists, please report this as in future the database will be required: {e}")
+ logging.error(
+ f"Failed to initialize database: {e}\n"
+ "The --enable-assets flag requires a working database connection.\n"
+ "To resolve this, try one of the following:\n"
+ " 1. Install the latest requirements: pip install -r requirements.txt\n"
+ " 2. Specify an alternative database URL: --database-url sqlite:///path/to/your.db\n"
+ " 3. Use an in-memory database: --database-url sqlite:///:memory:"
+ )
+ sys.exit(1)
+
+
+def warn_if_database_in_use():
+ db_path = lock_holder_db_path()
+ if db_path is None:
+ return
+ app.logger.log_startup_warning(
+ f"""
+________________________________________________________________________
+WARNING WARNING WARNING WARNING WARNING
+
+Another ComfyUI is already using this install's asset database:
+ {db_path}
+This ComfyUI was started without --enable-assets, so it doesn't need that database and will start anyway.
+A future version will refuse to start two ComfyUIs on the same asset database.
+To run both, give this one its own:
+ --database-url sqlite:///path/to/another.db
+________________________________________________________________________
+""".strip()
+ )
def start_comfyui(asyncio_loop=None):
diff --git a/tests-unit/app_test/test_db_assets_off.py b/tests-unit/app_test/test_db_assets_off.py
new file mode 100644
index 000000000..8a08c8177
--- /dev/null
+++ b/tests-unit/app_test/test_db_assets_off.py
@@ -0,0 +1,142 @@
+import os
+
+import pytest
+import torch
+from filelock import FileLock, Timeout
+
+import app.logger
+from app.database import db as db_module
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+import main # noqa: E402
+
+
+def db_path(tmp_path, monkeypatch):
+ path = str(tmp_path / "comfyui.db")
+ monkeypatch.setattr(db_module.args, "database_url", f"sqlite:///{path}")
+ return path
+
+
+def startup_warnings(monkeypatch):
+ warnings = []
+ monkeypatch.setattr(app.logger, "STARTUP_WARNINGS", warnings)
+ return warnings
+
+
+class _AssetsOff:
+ enabled = False
+
+ def __init__(self):
+ self.started = False
+
+ def startup(self):
+ self.started = True
+
+
+def test_probe_without_a_lock_file_creates_none(db_path):
+ assert db_module.lock_holder_db_path() is None
+ assert not os.path.exists(db_path + ".lock")
+
+
+def test_probe_reports_a_held_lock(db_path):
+ holder = FileLock(db_path + ".lock")
+ holder.acquire(timeout=0)
+ try:
+ assert db_module.lock_holder_db_path() == db_path
+ finally:
+ holder.release()
+
+
+def test_probe_does_not_keep_a_free_lock(db_path, monkeypatch):
+ open(db_path + ".lock", "a").close()
+ probes = []
+
+ class _KeptAlive(FileLock):
+ # Holding a reference stops garbage collection from releasing a lock the probe kept.
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ probes.append(self)
+
+ monkeypatch.setattr(db_module, "FileLock", _KeptAlive)
+
+ assert db_module.lock_holder_db_path() is None
+ assert len(probes) == 1
+
+ contender = FileLock(db_path + ".lock")
+ try:
+ contender.acquire(timeout=0)
+ except Timeout:
+ pytest.fail("the probe kept the lock it was only meant to test")
+ contender.release()
+
+
+def test_probe_skips_non_sqlite_databases(monkeypatch):
+ monkeypatch.setattr(db_module.args, "database_url", "postgresql://localhost/comfy")
+
+ assert db_module.lock_holder_db_path() is None
+
+
+def test_probe_treats_a_failed_check_as_free(db_path, monkeypatch):
+ open(db_path + ".lock", "a").close()
+
+ def _unexpected(self, *args, **kwargs):
+ raise NotImplementedError("no locking on this filesystem")
+
+ monkeypatch.setattr(FileLock, "acquire", _unexpected)
+
+ assert db_module.lock_holder_db_path() is None
+
+
+def test_assets_off_leaves_the_database_alone(db_path, startup_warnings, monkeypatch):
+ monkeypatch.setattr(main, "dependencies_available", lambda: True)
+
+ def _init_db():
+ pytest.fail("init_db ran with assets off")
+
+ monkeypatch.setattr(main, "init_db", _init_db)
+ asset_manager = _AssetsOff()
+
+ main.setup_database(asset_manager)
+
+ assert asset_manager.started
+ assert not os.path.exists(db_path)
+ assert not os.path.exists(db_path + ".lock")
+ assert startup_warnings == []
+
+
+def test_assets_off_warns_and_continues_when_another_process_holds_the_lock(
+ db_path, startup_warnings, monkeypatch
+):
+ monkeypatch.setattr(main, "dependencies_available", lambda: True)
+ monkeypatch.setattr(main, "init_db", lambda: pytest.fail("init_db ran with assets off"))
+ asset_manager = _AssetsOff()
+ holder = FileLock(db_path + ".lock")
+ holder.acquire(timeout=0)
+ try:
+ main.setup_database(asset_manager)
+ finally:
+ holder.release()
+
+ assert asset_manager.started
+ assert len(startup_warnings) == 1
+ warning = startup_warnings[0]
+ assert "Another ComfyUI is already using this install's asset database" in warning
+ assert db_path in warning
+ assert "This ComfyUI was started without --enable-assets, so it doesn't need that database and will start anyway" in warning
+ assert "A future version will refuse to start two ComfyUIs on the same asset database" in warning
+
+
+def test_probe_treats_a_failed_release_as_free(db_path, monkeypatch):
+ open(db_path + ".lock", "a").close()
+
+ def _unexpected(self, *args, **kwargs):
+ raise OSError("unlock failed")
+
+ monkeypatch.setattr(FileLock, "release", _unexpected)
+
+ assert db_module.lock_holder_db_path() is None
diff --git a/tests-unit/app_test/test_db_init_locking.py b/tests-unit/app_test/test_db_init_locking.py
index c31065432..b4aa9bca3 100644
--- a/tests-unit/app_test/test_db_init_locking.py
+++ b/tests-unit/app_test/test_db_init_locking.py
@@ -76,9 +76,8 @@ def test_failed_init_releases_the_lock(stale_db, monkeypatch):
contender.acquire(timeout=0)
except Timeout:
pytest.fail(
- "a failed init stranded the lock; setup_database logs and CONTINUES when assets are "
- "disabled, so this process would block every other instance for its whole lifetime "
- "over a database it never opened"
+ "a failed init stranded the lock, so this process would block every other "
+ "instance for its whole lifetime over a database it never opened"
)
contender.release()
@@ -135,25 +134,11 @@ def test_legacy_database_copy_runs_under_file_lock(tmp_path, monkeypatch):
assert copied == [(str(legacy_db) + ".bak", str(target_db))]
-def test_setup_database_routes_file_lock_to_lock_guidance(monkeypatch, caplog):
- monkeypatch.setattr(main, "dependencies_available", lambda: True)
-
- def _raise_file_lock():
- raise RuntimeError(
- "Could not acquire lock on database 'x.db'. "
- "Another ComfyUI process may already be using it. "
- "Use --database-url to specify a separate database file."
- )
-
- monkeypatch.setattr(main, "init_db", _raise_file_lock)
- monkeypatch.setattr(main.args, "enable_assets", False)
-
- with caplog.at_level(logging.ERROR):
- result = main.setup_database(None)
+class _AssetsOn:
+ enabled = True
- assert result is None
- assert "Database is locked. Another ComfyUI process is already using this database." in caplog.text
- assert "Failed to initialize database." not in caplog.text
+ def startup(self):
+ pass
def test_setup_database_exits_for_file_lock_when_assets_are_enabled(monkeypatch, caplog):
@@ -167,32 +152,46 @@ def test_setup_database_exits_for_file_lock_when_assets_are_enabled(monkeypatch,
)
monkeypatch.setattr(main, "init_db", _raise_file_lock)
- monkeypatch.setattr(main.args, "enable_assets", True)
with caplog.at_level(logging.ERROR), pytest.raises(SystemExit) as error:
- main.setup_database(None)
+ main.setup_database(_AssetsOn())
assert error.value.code == 1
assert "Database is locked. Another ComfyUI process is already using this database." in caplog.text
assert "The --enable-assets flag requires a working database connection." not in caplog.text
-def test_setup_database_exits_for_driver_lock_when_assets_are_disabled(monkeypatch, caplog):
+def test_setup_database_exits_for_driver_lock_when_assets_are_enabled(monkeypatch, caplog):
monkeypatch.setattr(main, "dependencies_available", lambda: True)
def _raise_driver_lock():
raise Exception("sqlite3.OperationalError: database is locked")
monkeypatch.setattr(main, "init_db", _raise_driver_lock)
- monkeypatch.setattr(main.args, "enable_assets", False)
with caplog.at_level(logging.ERROR), pytest.raises(SystemExit) as error:
- main.setup_database(None)
+ main.setup_database(_AssetsOn())
assert error.value.code == 1
assert "Database is locked. Another ComfyUI process is already using this database." in caplog.text
+def test_setup_database_exits_for_other_failures_when_assets_are_enabled(monkeypatch, caplog):
+ monkeypatch.setattr(main, "dependencies_available", lambda: True)
+
+ def _raise_other():
+ raise RuntimeError("alembic config exploded")
+
+ monkeypatch.setattr(main, "init_db", _raise_other)
+
+ with caplog.at_level(logging.ERROR), pytest.raises(SystemExit) as error:
+ main.setup_database(_AssetsOn())
+
+ assert error.value.code == 1
+ assert "The --enable-assets flag requires a working database connection." in caplog.text
+ assert "Database is locked." not in caplog.text
+
+
def test_failed_restore_does_not_mask_the_upgrade_error(stale_db, monkeypatch, caplog):
real_backup = db_module._backup_database
diff --git a/tests-unit/app_test/test_db_lock_processes.py b/tests-unit/app_test/test_db_lock_processes.py
new file mode 100644
index 000000000..41cb0a794
--- /dev/null
+++ b/tests-unit/app_test/test_db_lock_processes.py
@@ -0,0 +1,133 @@
+"""Real ComfyUI processes sharing one install's database: an assets-off instance must
+not block an assets-on one, and must warn when another process holds the database."""
+
+import socket
+import subprocess
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+LOCK_HELD = "Database is locked. Another ComfyUI process is already using this database."
+IN_USE = "Another ComfyUI is already using this install's asset database"
+
+HOLD_SCRIPT = (
+ "import sys; "
+ "from filelock import FileLock; "
+ "lock = FileLock(sys.argv[1]); lock.acquire(timeout=0); "
+ "print('held', flush=True); "
+ "sys.stdin.readline()"
+)
+
+
+def _comfy_args(base: Path, *flags: str) -> list[str]:
+ return [
+ sys.executable,
+ "main.py",
+ "--cpu",
+ "--disable-all-custom-nodes",
+ "--disable-partner-nodes",
+ f"--base-directory={base}",
+ f"--front-end-root={base}",
+ f"--database-url=sqlite:///{base / 'comfyui.db'}",
+ *flags,
+ ]
+
+
+def _quick_start(base: Path, *flags: str) -> subprocess.CompletedProcess:
+ return subprocess.run(
+ _comfy_args(base, "--quick-test-for-ci", *flags),
+ cwd=REPO_ROOT,
+ capture_output=True,
+ text=True,
+ timeout=120,
+ )
+
+
+def _free_port() -> int:
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _wait_until_serving(proc: subprocess.Popen, port: int, timeout: float = 120.0) -> None:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if proc.poll() is not None:
+ raise AssertionError(f"ComfyUI exited with code {proc.returncode} before serving")
+ try:
+ with urllib.request.urlopen(f"http://127.0.0.1:{port}/system_stats", timeout=2) as response:
+ if response.status == 200:
+ return
+ except OSError:
+ pass
+ time.sleep(0.2)
+ raise AssertionError("ComfyUI did not start serving")
+
+
+def held_lock(tmp_path):
+ holder = subprocess.Popen(
+ [sys.executable, "-c", HOLD_SCRIPT, str(tmp_path / "comfyui.db.lock")],
+ cwd=REPO_ROOT,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ text=True,
+ )
+ try:
+ assert holder.stdout.readline().strip() == "held"
+ yield
+ finally:
+ holder.kill()
+ holder.wait()
+
+
+def test_running_assets_off_instance_does_not_block_an_assets_on_start(tmp_path):
+ # A previous assets-on run leaves the database and its lock file behind, as on a real install.
+ first = _quick_start(tmp_path, "--enable-assets")
+ assert first.returncode == 0, first.stderr
+
+ port = _free_port()
+ server_log = tmp_path / "server.log"
+ with open(server_log, "w") as log:
+ server = subprocess.Popen(
+ _comfy_args(tmp_path, "--listen", "127.0.0.1", "--port", str(port)),
+ cwd=REPO_ROOT,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ )
+ try:
+ try:
+ _wait_until_serving(server, port)
+ except AssertionError as e:
+ raise AssertionError(f"{e}\n{server_log.read_text()[-4000:]}") from None
+
+ second = _quick_start(tmp_path, "--enable-assets")
+
+ assert server.poll() is None
+ finally:
+ server.terminate()
+ server.wait(timeout=30)
+
+ assert second.returncode == 0, second.stderr
+ assert LOCK_HELD not in second.stderr
+
+
+def test_assets_off_start_warns_and_continues_when_the_database_is_held(tmp_path, held_lock):
+ result = _quick_start(tmp_path)
+
+ assert result.returncode == 0, result.stderr
+ assert IN_USE in result.stderr
+ assert str(tmp_path / "comfyui.db") in result.stderr
+ assert "A future version will refuse to start two ComfyUIs on the same asset database" in result.stderr
+ assert "Traceback" not in result.stderr
+
+
+def test_assets_on_start_still_fails_when_the_database_is_held(tmp_path, held_lock):
+ result = _quick_start(tmp_path, "--enable-assets")
+
+ assert result.returncode == 1, result.stderr
+ assert LOCK_HELD in result.stderr
diff --git a/tests-unit/assets_test/services/test_noassets_parity.py b/tests-unit/assets_test/services/test_noassets_parity.py
index 121582f95..e1ff69ac4 100644
--- a/tests-unit/assets_test/services/test_noassets_parity.py
+++ b/tests-unit/assets_test/services/test_noassets_parity.py
@@ -49,7 +49,7 @@ async def test_noassets_register_routes_returns_service_disabled_and_disables_se
assert asset_seeder.is_disabled()
-def test_noassets_startup_applies_hash_mode_persists_state_and_cleans_temp_dir(
+def test_noassets_startup_applies_hash_mode_leaves_database_alone_and_cleans_temp_dir(
mock_create_session: Callable[[], AbstractContextManager[Session]],
session: Session,
tmp_path: Path,
@@ -63,7 +63,7 @@ def test_noassets_startup_applies_hash_mode_persists_state_and_cleans_temp_dir(
_no_assets(hashing=True).startup()
assert hashing_enabled() is True
- assert read_stored_mode(session) == "on"
+ assert read_stored_mode(session) is None
assert not temp_file.exists()