Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(host-agent): build bundled images in prepare, not inside the start (#8248)

Mike Bradleyfb9cd33de668
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.py
index a1faf23dd..78f228210 100755
--- a/ods/bin/ods-host-agent.py
+++ b/ods/bin/ods-host-agent.py
@@ -9577,33 +9577,42 @@ def _image_prepare_context(service_ids: list[str]) -> tuple[list[str], dict[str,
def _services_missing_images(flags: list[str], service_ids: list[str],
- env: dict[str, str]) -> list[str]:
+ env: dict[str, str]) -> tuple[list[str], list[str]]:
"""The services among ``service_ids`` whose Compose image is not on this host.
- Locally built images are left to their own build step. The resolved
- configuration carries credential values, so it is parsed and never logged.
+ Returns ``(to_pull, to_build)``. A locally built service is checked under
+ the name ``up`` gives its image: its ``image``, else Compose's
+ ``<project>-<service>``. The resolved configuration carries credential
+ values, so it is parsed and never logged.
"""
command = ["docker", "compose", *flags, "config", "--format", "json"]
result = subprocess.run(command, cwd=str(INSTALL_DIR), env=env,
capture_output=True, text=True, timeout=60)
if result.returncode:
raise RuntimeError("Could not resolve the Compose configuration for these services")
- services = json.loads(result.stdout).get("services")
+ config = json.loads(result.stdout)
+ services = config.get("services")
if not isinstance(services, dict):
raise RuntimeError("Invalid Compose configuration")
- missing = []
+ pulls, builds = [], []
for service_id in service_ids:
definition = services.get(service_id)
if not isinstance(definition, dict):
raise RuntimeError(f"The Compose stack does not define {service_id}")
+ built = bool(definition.get("build"))
image = definition.get("image")
- if definition.get("build") or not isinstance(image, str) or not image:
+ if built and not image:
+ project = config.get("name")
+ if not isinstance(project, str) or not project:
+ raise RuntimeError("Invalid Compose configuration")
+ image = f"{project}-{service_id}"
+ if not isinstance(image, str) or not image:
continue
inspected = subprocess.run(["docker", "image", "inspect", image],
capture_output=True, text=True, timeout=30)
if inspected.returncode:
- missing.append(service_id)
- return missing
+ (builds if built else pulls).append(service_id)
+ return pulls, builds
def _missing_model_files(service_ids: list[str]) -> list:
@@ -9641,54 +9650,110 @@ def _fetch_model_files(progress_id: str, files: list) -> tuple[bool, str]:
return True, ""
-def _pull_compose_images(flags: list[str], service_ids: list[str], progress_id: str,
- *, env: dict[str, str] | None = None) -> tuple[bool, str]:
- """Download the images of ``service_ids`` with live progress under ``progress_id``.
+def _follow_image_command(command: list[str], progress_id: str, label: str, detail,
+ subject: str, *, env: dict[str, str] | None = None) -> tuple[int, str, list[str]]:
+ """Run a Compose image command with live progress under ``progress_id``.
- Compose resolves each image from the same files and pinned digests that
- ``up`` uses. Returns ``(ok, error)``; the pull is stopped only when Docker
- reports nothing for IMAGE_PULL_STALL_SECONDS or IMAGE_PULL_MAX_SECONDS pass.
+ ``detail(line)`` reads each output line and returns the progress detail to
+ show after the elapsed time, or None to keep the last one. The command is
+ stopped only when it prints nothing for IMAGE_PULL_STALL_SECONDS or
+ IMAGE_PULL_MAX_SECONDS pass. Returns ``(returncode, failure, tail)``:
+ ``failure`` says why the command was stopped, ``tail`` is its last output.
"""
- command = ["docker", "compose", "--progress", "plain", *flags, "pull", *service_ids]
proc = subprocess.Popen(command, cwd=str(INSTALL_DIR), env=env, text=True, errors="replace",
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
tail: collections.deque = collections.deque(maxlen=40)
- seen = {"last": time.monotonic(), "layers": 0}
+ seen = {"last": time.monotonic(), "detail": detail("") or ""}
def _read_output() -> None:
for line in proc.stdout:
tail.append(line.rstrip())
seen["last"] = time.monotonic()
- if "Pull complete" in line or "Already exists" in line:
- seen["layers"] += 1
+ seen["detail"] = detail(line) or seen["detail"]
reader = threading.Thread(target=_read_output, daemon=True)
reader.start()
started = reported = time.monotonic()
- _write_progress(progress_id, "pulling", "Downloading images...")
+ _write_progress(progress_id, "pulling", label)
failure = ""
while proc.poll() is None:
time.sleep(1)
now = time.monotonic()
if now - seen["last"] > IMAGE_PULL_STALL_SECONDS:
- failure = f"Image download made no progress for {IMAGE_PULL_STALL_SECONDS // 60} minutes."
+ failure = f"{subject} made no progress for {IMAGE_PULL_STALL_SECONDS // 60} minutes."
elif now - started > IMAGE_PULL_MAX_SECONDS:
- failure = f"Image download did not finish within {IMAGE_PULL_MAX_SECONDS // 3600} hours."
+ failure = f"{subject} did not finish within {IMAGE_PULL_MAX_SECONDS // 3600} hours."
if failure:
proc.kill()
break
if now - reported >= _IMAGE_PULL_PROGRESS_SECONDS:
reported = now
minutes, seconds = divmod(int(now - started), 60)
+ elapsed = f"{label} {minutes}:{seconds:02d} elapsed"
_write_progress(progress_id, "pulling",
- f"Downloading images... {minutes}:{seconds:02d} elapsed, "
- f"{seen['layers']} layers done")
+ f"{elapsed}, {seen['detail']}" if seen["detail"] else elapsed)
proc.wait()
reader.join(timeout=5)
+ return proc.returncode, failure, list(tail)
+
+
+def _pull_compose_images(flags: list[str], service_ids: list[str], progress_id: str,
+ *, env: dict[str, str] | None = None) -> tuple[bool, str]:
+ """Download the images of ``service_ids`` with live progress under ``progress_id``.
+
+ Compose resolves each image from the same files and pinned digests that
+ ``up`` uses. Returns ``(ok, error)``; the pull is stopped only when Docker
+ reports nothing for IMAGE_PULL_STALL_SECONDS or IMAGE_PULL_MAX_SECONDS pass.
+ """
+ command = ["docker", "compose", "--progress", "plain", *flags, "pull", *service_ids]
+ layers = [0]
+
+ def detail(line: str) -> str:
+ if "Pull complete" in line or "Already exists" in line:
+ layers[0] += 1
+ return f"{layers[0]} layers done"
+
+ returncode, failure, tail = _follow_image_command(
+ command, progress_id, "Downloading images...", detail, "Image download", env=env)
+ if failure:
+ return False, failure
+ if returncode:
+ return False, "Image download failed:\n" + "\n".join(tail[-12:])
+ return True, ""
+
+
+# Compose's plain build progress (Compose 5.2): "#14 [comfyui 10/15] RUN ..."
+# starts a Dockerfile step, "#20 [comfyui] exporting to image" saves the image.
+_BUILD_STEP_RE = re.compile(r"#\d+ \[(?:[^\]]* )?\s*(\d+)/(\d+)\]")
+_BUILD_EXPORT_RE = re.compile(r"#\d+ (?:\[[^\]]*\] )?exporting to ")
+
+
+def _build_compose_images(flags: list[str], service_ids: list[str], progress_id: str,
+ *, env: dict[str, str] | None = None) -> tuple[bool, str]:
+ """Build the local images of ``service_ids`` with live progress under ``progress_id``.
+
+ Compose builds from the same files ``up`` uses, so ``up`` then only
+ creates the container. A first ComfyUI build for NVIDIA installs its
+ Python stack and saves a 7 GB image: ten minutes on Workstation06
+ (2026-10-26), all of the 600 s start allowance, so its first ``up`` timed
+ out. Returns ``(ok, error)``; the build is stopped only when BuildKit
+ prints nothing for IMAGE_PULL_STALL_SECONDS or IMAGE_PULL_MAX_SECONDS pass.
+ """
+ command = ["docker", "compose", "--progress", "plain", *flags, "build", *service_ids]
+
+ def detail(line: str) -> str | None:
+ if _BUILD_EXPORT_RE.match(line):
+ return "saving the image"
+ step = _BUILD_STEP_RE.match(line)
+ return f"step {step[1]} of {step[2]}" if step else None
+
+ returncode, failure, tail = _follow_image_command(
+ command, progress_id, "Building images...", detail, "Image build", env=env)
if failure:
return False, failure
- if proc.returncode:
- return False, "Image download failed:\n" + "\n".join(list(tail)[-12:])
+ if returncode:
+ return False, _install_build_diagnostic(
+ subprocess.CompletedProcess(command, returncode, "\n".join(tail), ""), {})
return True, ""
@@ -13465,11 +13530,13 @@ class AgentHandler(BaseHTTPRequestHandler):
"""Download the images and models a planned enable needs before anything is selected.
Takes service ids only: bundled extensions and Open WebUI, whose images
- Compose resolves from their shipped files and pinned digests. Their
- default model packs (model-packs.json) follow the images. Answers 200
- when everything is already here, otherwise 202 and downloads in the
- background, reporting progress under ``progress_id`` until it records
- ``prepared`` or ``error``. Each service allows one operation.
+ Compose resolves from their shipped files and pinned digests. Images a
+ service builds from its shipped Dockerfile are built here too, so the
+ start that follows only creates containers. Their default model packs
+ (model-packs.json) follow the images. Answers 200 when everything is
+ already here, otherwise 202 and works in the background, reporting
+ progress under ``progress_id`` until it records ``prepared`` or
+ ``error``. Each service allows one operation.
"""
if not check_auth(self):
return
@@ -13504,32 +13571,40 @@ class AgentHandler(BaseHTTPRequestHandler):
try:
flags, env = _image_prepare_context(service_ids)
- missing = _services_missing_images(flags, service_ids, env)
+ pulls, builds = _services_missing_images(flags, service_ids, env)
model_files = _missing_model_files(service_ids)
except (RuntimeError, ValueError, OSError, subprocess.SubprocessError) as exc:
_release()
logger.warning("Image preparation could not resolve %s: %s", progress_id, type(exc).__name__)
json_response(self, 503, {"error": str(exc)[:300]})
return
- if not missing and not model_files:
+ if not pulls and not builds and not model_files:
_release()
json_response(self, 200, {"status": "ready", "service_ids": service_ids})
return
def _download() -> None:
try:
- if missing:
- pulled, error = _pull_compose_images(flags, missing, progress_id, env=env)
+ if pulls:
+ pulled, error = _pull_compose_images(flags, pulls, progress_id, env=env)
if not pulled:
_write_progress(progress_id, "error", "Image download failed", error=error)
return
+ if builds:
+ built, error = _build_compose_images(flags, builds, progress_id, env=env)
+ if not built:
+ _write_progress(progress_id, "error", "Image build failed", error=error)
+ return
if model_files:
fetched, error = _fetch_model_files(progress_id, model_files)
if not fetched:
_write_progress(progress_id, "error", "Model download failed", error=error)
return
- _write_progress(progress_id, "prepared",
- "Images and models downloaded" if model_files else "Images downloaded")
+ if builds:
+ label = "Images built and models downloaded" if model_files else "Images built"
+ else:
+ label = "Images and models downloaded" if model_files else "Images downloaded"
+ _write_progress(progress_id, "prepared", label)
except (OSError, ValueError, subprocess.SubprocessError) as exc:
logger.exception("Image preparation failed for %s", progress_id)
_write_progress(progress_id, "error", "Image download failed", error=str(exc)[:500])
@@ -13541,8 +13616,8 @@ class AgentHandler(BaseHTTPRequestHandler):
except Exception:
_release()
raise
- json_response(self, 202, {"status": "accepted", "service_ids": service_ids, "pulling": missing,
- "models": [item.target.name for item in model_files]})
+ json_response(self, 202, {"status": "accepted", "service_ids": service_ids, "pulling": pulls,
+ "building": builds, "models": [item.target.name for item in model_files]})
# ── Model management handlers ──
diff --git a/ods/docs/HOST-AGENT-API.md b/ods/docs/HOST-AGENT-API.md
index 419ff9389..8ea7d662e 100644
--- a/ods/docs/HOST-AGENT-API.md
+++ b/ods/docs/HOST-AGENT-API.md
@@ -290,12 +290,14 @@ Compose resolves the images from the services' shipped files and pinned digests.
{"status": "ready", "service_ids": ["searxng", "hermes", "hermes-proxy"]}
```
-**Response (202)** while the missing images download in the background:
+**Response (202)** while the missing images download or build in the background:
```json
-{"status": "accepted", "service_ids": ["searxng", "hermes", "hermes-proxy"], "pulling": ["hermes"]}
+{"status": "accepted", "service_ids": ["searxng", "hermes", "hermes-proxy"], "pulling": ["hermes"], "building": [], "models": []}
```
-The progress record under `progress_id` shows `pulling`, with an elapsed time and completed layer count in `phase_label`, then `prepared` or `error`. The download stops only when Docker reports nothing for `ODS_IMAGE_PULL_STALL_SECONDS` (default 900) or after `ODS_IMAGE_PULL_MAX_SECONDS` (default 21600), not after a fixed request timeout. The Library install path downloads its images the same way.
+`building` lists the services whose image Compose builds from the shipped Dockerfile (a `build:` section, such as ComfyUI on NVIDIA) and is not on the host yet, checked under the `image` name or Compose's `<project>-<service>` default. They are built after the downloads, so the start that follows only creates containers; a first ComfyUI build takes about ten minutes, longer than the 600 s start allowance.
+
+The progress record under `progress_id` shows `pulling`, with an elapsed time and completed layer count (downloads) or the current Dockerfile step (builds) in `phase_label`, then `prepared` or `error`. A download or build stops only when Docker reports nothing for `ODS_IMAGE_PULL_STALL_SECONDS` (default 900) or after `ODS_IMAGE_PULL_MAX_SECONDS` (default 21600), not after a fixed request timeout. The Library install path downloads its images the same way.
**Error responses:**
| Code | Condition |
diff --git a/ods/extensions/services/dashboard-api/tests/test_host_agent_image_prepare.py b/ods/extensions/services/dashboard-api/tests/test_host_agent_image_prepare.py
index cc3d2981b..f3a89a189 100644
--- a/ods/extensions/services/dashboard-api/tests/test_host_agent_image_prepare.py
+++ b/ods/extensions/services/dashboard-api/tests/test_host_agent_image_prepare.py
@@ -1,4 +1,4 @@
-"""Image downloads that outlast a request: prepare-images and the streaming pull.
+"""Image work that outlasts a request: prepare-images, the streaming pull and build.
On the Mac mini's slow link (2026-10-04) Hermes' first `docker compose up`
spent its whole 600 s start allowance downloading, and adding Open WebUI died
@@ -6,6 +6,12 @@ at 600 s with the 0.11.4 image unfinished, while the Dashboard had given up at
180 s. Bundled services now download their images first through
/v1/extension/prepare-images, before anything is selected, with progress on
the card; the pull stops only when Docker stops making progress.
+
+On Workstation06 (2026-10-26) ComfyUI's first NVIDIA image build took the
+whole 600 s start allowance (ten minutes, 0% cached), so the start timed out
+and the card showed ComfyUI stopped. Images a bundled service builds from its
+shipped Dockerfile are now built in the same step, with the build step on the
+card.
"""
import json
@@ -116,12 +122,93 @@ def test_failed_pull_returns_dockers_last_words(host, monkeypatch, clock): # no
assert error.splitlines() == ["Image download failed:", " Image x Pulling", "toomanyrequests: rate limit"]
-def test_missing_images_skips_local_builds_and_names_the_services(host, monkeypatch): # noqa: F811
- config = {"services": {
+class TimedBuild(FakePull):
+ """`docker compose build`: prints its lines at once, then runs until the faked clock reaches ``exit_at``."""
+
+ def __init__(self, command, clock, exit_at, lines=(), returncode=0):
+ super().__init__(command, lines=lines, returncode=returncode)
+ self._clock = clock
+ self._exit_at = exit_at
+
+ def poll(self):
+ if self._clock[0] < self._exit_at and not self.killed:
+ return None
+ return super().poll()
+
+
[email protected]("lines, detail", [
+ (["#5 [comfyui 1/15] FROM docker.io/nvidia/cuda:12.8.0-runtime-ubuntu22.04\n",
+ "#14 [comfyui 10/15] RUN pip3 install --no-cache-dir comfy-kitchen\n",
+ "#14 12.31 Collecting comfy-kitchen\n"], "step 10 of 15"),
+ # Compose 5.2 names the target in the export line; its sub-lines do not.
+ (["#19 [comfyui 15/15] WORKDIR /opt/comfyui\n", "#20 [comfyui] exporting to image\n",
+ "#20 exporting layers 125.3s done\n"], "saving the image"),
+])
+def test_build_reports_its_step_and_uses_the_compose_files_up_uses(host, monkeypatch, clock, lines, detail): # noqa: F811
+ started = []
+
+ def popen(command, **options):
+ started.append((list(command), options))
+ return TimedBuild(command, clock, clock[0] + 40, lines=lines)
+
+ monkeypatch.setattr(_mod.subprocess, "Popen", popen)
+ written = []
+ monkeypatch.setattr(_mod, "_write_progress", lambda *args, **kwargs: written.append(args))
+
+ ok, error = _mod._build_compose_images(["-f", "base.yml"], ["comfyui"], "comfyui")
+
+ assert (ok, error) == (True, "")
+ command, options = started[0]
+ assert command == ["docker", "compose", "--progress", "plain", "-f", "base.yml", "build", "comfyui"]
+ assert options["stderr"] is subprocess.STDOUT
+ labels = [args[2] for args in written if args[:2] == ("comfyui", "pulling")]
+ assert labels[0] == "Building images..."
+ assert f"Building images... 0:15 elapsed, {detail}" in labels
+
+
+def test_build_stops_when_buildkit_reports_nothing(host, monkeypatch, clock): # noqa: F811
+ monkeypatch.setattr(_mod, "IMAGE_PULL_STALL_SECONDS", 120)
+ monkeypatch.setattr(_mod, "IMAGE_PULL_MAX_SECONDS", 10 ** 6)
+ build = FakePull(["cmd"], hang=True)
+ monkeypatch.setattr(_mod.subprocess, "Popen", lambda command, **options: build)
+ written = []
+ monkeypatch.setattr(_mod, "_write_progress", lambda *args, **kwargs: written.append(args))
+
+ ok, error = _mod._build_compose_images([], ["comfyui"], "comfyui")
+
+ assert ok is False and build.killed
+ assert error == "Image build made no progress for 2 minutes."
+ # Before the first Dockerfile step there is no step to name.
+ assert ("comfyui", "pulling", "Building images... 0:15 elapsed") in written
+
+
+def test_failed_build_leads_with_buildkits_decisive_error_without_credentials(host, monkeypatch, clock): # noqa: F811
+ _popen(monkeypatch, returncode=1, lines=[
+ "#14 [comfyui 10/15] RUN pip3 install comfy-kitchen\n",
+ "#14 0.52 token SERVICE_API_KEY=persisted-credential\n",
+ "#14 ERROR: process did not complete successfully: exit code: 1\n",
+ 'failed to solve: process "/bin/sh -c pip3 install comfy-kitchen" did not complete successfully: exit code: 1\n',
+ ])
+ monkeypatch.setattr(_mod, "_write_progress", lambda *args, **kwargs: None)
+
+ ok, error = _mod._build_compose_images([], ["comfyui"], "comfyui")
+
+ assert ok is False
+ assert error.startswith('Untrusted build error: failed to solve: process "/bin/sh -c pip3 install')
+ assert "#14 [comfyui 10/15] RUN pip3 install comfy-kitchen" in error
+ assert "persisted-credential" not in error
+
+
+def test_missing_images_names_pulls_and_local_builds_apart(host, monkeypatch): # noqa: F811
+ config = {"name": "ods", "services": {
"hermes": {"image": "nousresearch/hermes-agent@sha256:" + "a" * 64},
"hermes-proxy": {"image": "caddy:2"},
"builder": {"image": "ods-built:local", "build": {"context": "."}},
+ # No image: Compose names what `up` builds <project>-<service>.
+ "comfyui": {"build": {"context": "./extensions/services/comfyui"}},
+ "token-spy": {"build": {"context": "./extensions/services/token-spy"}},
}}
+ present = {"caddy:2", "ods-token-spy"}
seen = []
def run(command, **kwargs):
@@ -130,20 +217,26 @@ def test_missing_images_skips_local_builds_and_names_the_services(host, monkeypa
assert kwargs["env"] == {"ENABLE_OPEN_WEBUI": "true"}
return types.SimpleNamespace(returncode=0, stdout=json.dumps(config), stderr="")
if command[:3] == ["docker", "image", "inspect"]:
- return types.SimpleNamespace(returncode=0 if command[3] == "caddy:2" else 1, stdout="", stderr="")
+ return types.SimpleNamespace(returncode=0 if command[3] in present else 1, stdout="", stderr="")
raise AssertionError(command)
monkeypatch.setattr(_mod.subprocess, "run", run)
- missing = _mod._services_missing_images(["-f", "b.yml"], ["hermes", "hermes-proxy", "builder"],
- {"ENABLE_OPEN_WEBUI": "true"})
- assert missing == ["hermes"]
- assert ["docker", "image", "inspect", "ods-built:local"] not in seen
+ pulls, builds = _mod._services_missing_images(
+ ["-f", "b.yml"], ["hermes", "hermes-proxy", "builder", "comfyui", "token-spy"],
+ {"ENABLE_OPEN_WEBUI": "true"})
+ assert (pulls, builds) == (["hermes"], ["builder", "comfyui"])
+ assert ["docker", "image", "inspect", "ods-comfyui"] in seen
-def test_missing_images_refuses_a_service_the_stack_does_not_define(host, monkeypatch): # noqa: F811
[email protected]("config, message", [
+ ({"name": "ods", "services": {}}, "does not define hermes"),
+ # Without the project name a build-only service's image cannot be named.
+ ({"services": {"hermes": {"build": {"context": "."}}}}, "Invalid Compose configuration"),
+])
+def test_missing_images_refuses_a_configuration_it_cannot_read(host, monkeypatch, config, message): # noqa: F811
monkeypatch.setattr(_mod.subprocess, "run", lambda command, **kwargs: types.SimpleNamespace(
- returncode=0, stdout=json.dumps({"services": {}}), stderr=""))
- with pytest.raises(RuntimeError, match="does not define hermes"):
+ returncode=0, stdout=json.dumps(config), stderr=""))
+ with pytest.raises(RuntimeError, match=message):
_mod._services_missing_images([], ["hermes"], {})
@@ -212,7 +305,7 @@ def test_prepare_refuses_library_and_unknown_services(prepare, host): # noqa: F
def test_prepare_answers_ready_without_downloading_when_images_are_here(prepare, host, monkeypatch): # noqa: F811
- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))
monkeypatch.setattr(_mod, "_pull_compose_images", lambda *args, **kwargs: pytest.fail("no download expected"))
status, response = prepare({"service_ids": ["hermes", "hermes-proxy"], "progress_id": "hermes"})
@@ -222,7 +315,7 @@ def test_prepare_answers_ready_without_downloading_when_images_are_here(prepare,
def test_prepare_downloads_in_the_background_and_records_the_result(prepare, host, monkeypatch): # noqa: F811
- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ["hermes"])
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: (["hermes"], []))
pulls = []
def pull(flags, services, progress_id, *, env=None):
@@ -242,7 +335,7 @@ def test_prepare_downloads_in_the_background_and_records_the_result(prepare, hos
def test_prepare_records_a_failed_download_for_the_card(prepare, host, monkeypatch): # noqa: F811
- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ["open-webui"])
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: (["open-webui"], []))
monkeypatch.setattr(_mod, "_pull_compose_images",
lambda *args, **kwargs: (False, "Image download made no progress for 15 minutes."))
@@ -255,6 +348,22 @@ def test_prepare_records_a_failed_download_for_the_card(prepare, host, monkeypat
assert not _mod._service_locks["open-webui"].locked()
+def test_prepare_records_a_failed_build_for_the_card(prepare, host, monkeypatch): # noqa: F811
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], ["hermes"]))
+ monkeypatch.setattr(_mod, "_pull_compose_images", lambda *args, **kwargs: pytest.fail("nothing to download"))
+ monkeypatch.setattr(_mod, "_build_compose_images",
+ lambda *args, **kwargs: (False, "Untrusted build error: failed to solve"))
+
+ status, response = prepare({"service_ids": ["hermes"], "progress_id": "hermes"})
+
+ # A missing local image alone is not ready: `up` would build it inside the start.
+ assert status == 202 and (response["pulling"], response["building"]) == ([], ["hermes"])
+ record = host.progress("hermes")
+ assert (record["status"], record["phase_label"]) == ("error", "Image build failed")
+ assert record["error"] == "Untrusted build error: failed to solve"
+ assert not _mod._service_locks["hermes"].locked()
+
+
def test_prepare_refuses_while_the_service_is_being_changed(prepare, host, monkeypatch): # noqa: F811
monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: pytest.fail("not reached"))
_mod._service_locks["hermes-proxy"].acquire()
@@ -291,7 +400,7 @@ def model_pack(prepare, host, monkeypatch): # noqa: F811
def test_prepare_fetches_a_missing_model_pack_after_the_images(prepare, host, monkeypatch, model_pack): # noqa: F811
- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ["painter"])
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: (["painter"], []))
order = []
def pull(*args, **kwargs):
@@ -299,6 +408,7 @@ def test_prepare_fetches_a_missing_model_pack_after_the_images(prepare, host, mo
return True, ""
monkeypatch.setattr(_mod, "_pull_compose_images", pull)
+ monkeypatch.setattr(_mod, "_build_compose_images", lambda *args, **kwargs: pytest.fail("nothing to build"))
def fetch(item, report):
order.append("model")
@@ -319,8 +429,42 @@ def test_prepare_fetches_a_missing_model_pack_after_the_images(prepare, host, mo
assert not _mod._service_locks["painter"].locked()
+def test_prepare_builds_local_images_after_downloads_and_before_models(prepare, host, monkeypatch, model_pack): # noqa: F811
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: (["hermes"], ["painter"]))
+ order = []
+
+ def pull(flags, services, progress_id, *, env=None):
+ order.append(("pull", services))
+ return True, ""
+
+ def build(flags, services, progress_id, *, env=None):
+ order.append(("build", services))
+ # The files and selectors the enable will run, under the card's record.
+ assert (flags, progress_id, env) == (["-f", "base.yml"], "painter", {"E": "1"})
+ assert _mod._service_locks["painter"].locked() # one operation per service
+ return True, ""
+
+ def fetch(item, report):
+ order.append(("model", item.target.name))
+ item.target.parent.mkdir(parents=True)
+ item.target.write_bytes(b"model")
+
+ monkeypatch.setattr(_mod, "_pull_compose_images", pull)
+ monkeypatch.setattr(_mod, "_build_compose_images", build)
+ monkeypatch.setattr(_mod._extension_assets, "fetch", fetch)
+
+ status, response = prepare({"service_ids": ["hermes", "painter"], "progress_id": "painter"})
+
+ assert status == 202
+ assert (response["pulling"], response["building"]) == (["hermes"], ["painter"])
+ assert order == [("pull", ["hermes"]), ("build", ["painter"]), ("model", PACK_FILE)]
+ record = host.progress("painter")
+ assert (record["status"], record["phase_label"]) == ("prepared", "Images built and models downloaded")
+ assert not _mod._service_locks["painter"].locked()
+
+
def test_prepare_downloads_a_missing_model_even_when_the_images_are_here(prepare, host, monkeypatch, model_pack): # noqa: F811
- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))
monkeypatch.setattr(_mod, "_pull_compose_images", lambda *args, **kwargs: pytest.fail("no image download"))
labels = []
real_write = _mod._write_progress
@@ -343,7 +487,7 @@ def test_prepare_downloads_a_missing_model_even_when_the_images_are_here(prepare
def test_prepare_is_ready_when_images_and_models_are_here(prepare, host, monkeypatch, model_pack): # noqa: F811
model_pack.parent.mkdir(parents=True)
model_pack.write_bytes(b"model") # the pinned size: 5 bytes
- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))
monkeypatch.setattr(_mod._extension_assets, "fetch", lambda *args: pytest.fail("no download expected"))
status, response = prepare({"service_ids": ["painter"], "progress_id": "painter"})
@@ -353,7 +497,7 @@ def test_prepare_is_ready_when_images_and_models_are_here(prepare, host, monkeyp
def test_prepare_records_a_failed_model_download_for_the_card(prepare, host, monkeypatch, model_pack): # noqa: F811
- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))
def fail(item, report):
raise _mod._extension_assets.AssetError("The download of Painter model stopped (timed out).")
@@ -371,7 +515,7 @@ def test_prepare_records_a_failed_model_download_for_the_card(prepare, host, mon
def test_prepare_reports_a_malformed_pack_list_without_downloading(prepare, host, monkeypatch, model_pack): # noqa: F811
(host.builtins / "painter" / "model-packs.json").write_text('{"schema": "other"}', encoding="utf-8")
- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])
+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))
status, response = prepare({"service_ids": ["painter"], "progress_id": "painter"})