ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(host-agent): build bundled images in prepare, not inside the start (#8248)
Mike Bradleyfb9cd33de668
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.pyindex a1faf23dd..78f228210 100755--- a/ods/bin/ods-host-agent.py+++ b/ods/bin/ods-host-agent.py@@ -9577,33 +9577,42 @@ def _image_prepare_context(service_ids: list[str]) -> tuple[list[str], dict[str,def _services_missing_images(flags: list[str], service_ids: list[str],- env: dict[str, str]) -> list[str]:+ env: dict[str, str]) -> tuple[list[str], list[str]]:"""The services among ``service_ids`` whose Compose image is not on this host.- Locally built images are left to their own build step. The resolved- configuration carries credential values, so it is parsed and never logged.+ Returns ``(to_pull, to_build)``. A locally built service is checked under+ the name ``up`` gives its image: its ``image``, else Compose's+ ``<project>-<service>``. The resolved configuration carries credential+ values, so it is parsed and never logged."""command = ["docker", "compose", *flags, "config", "--format", "json"]result = subprocess.run(command, cwd=str(INSTALL_DIR), env=env,capture_output=True, text=True, timeout=60)if result.returncode:raise RuntimeError("Could not resolve the Compose configuration for these services")- services = json.loads(result.stdout).get("services")+ config = json.loads(result.stdout)+ services = config.get("services")if not isinstance(services, dict):raise RuntimeError("Invalid Compose configuration")- missing = []+ pulls, builds = [], []for service_id in service_ids:definition = services.get(service_id)if not isinstance(definition, dict):raise RuntimeError(f"The Compose stack does not define {service_id}")+ built = bool(definition.get("build"))image = definition.get("image")- if definition.get("build") or not isinstance(image, str) or not image:+ if built and not image:+ project = config.get("name")+ if not isinstance(project, str) or not project:+ raise RuntimeError("Invalid Compose configuration")+ image = f"{project}-{service_id}"+ if not isinstance(image, str) or not image:continueinspected = subprocess.run(["docker", "image", "inspect", image],capture_output=True, text=True, timeout=30)if inspected.returncode:- missing.append(service_id)- return missing+ (builds if built else pulls).append(service_id)+ return pulls, buildsdef _missing_model_files(service_ids: list[str]) -> list:@@ -9641,54 +9650,110 @@ def _fetch_model_files(progress_id: str, files: list) -> tuple[bool, str]:return True, ""-def _pull_compose_images(flags: list[str], service_ids: list[str], progress_id: str,- *, env: dict[str, str] | None = None) -> tuple[bool, str]:- """Download the images of ``service_ids`` with live progress under ``progress_id``.+def _follow_image_command(command: list[str], progress_id: str, label: str, detail,+ subject: str, *, env: dict[str, str] | None = None) -> tuple[int, str, list[str]]:+ """Run a Compose image command with live progress under ``progress_id``.- Compose resolves each image from the same files and pinned digests that- ``up`` uses. Returns ``(ok, error)``; the pull is stopped only when Docker- reports nothing for IMAGE_PULL_STALL_SECONDS or IMAGE_PULL_MAX_SECONDS pass.+ ``detail(line)`` reads each output line and returns the progress detail to+ show after the elapsed time, or None to keep the last one. The command is+ stopped only when it prints nothing for IMAGE_PULL_STALL_SECONDS or+ IMAGE_PULL_MAX_SECONDS pass. Returns ``(returncode, failure, tail)``:+ ``failure`` says why the command was stopped, ``tail`` is its last output."""- command = ["docker", "compose", "--progress", "plain", *flags, "pull", *service_ids]proc = subprocess.Popen(command, cwd=str(INSTALL_DIR), env=env, text=True, errors="replace",stdout=subprocess.PIPE, stderr=subprocess.STDOUT)tail: collections.deque = collections.deque(maxlen=40)- seen = {"last": time.monotonic(), "layers": 0}+ seen = {"last": time.monotonic(), "detail": detail("") or ""}def _read_output() -> None:for line in proc.stdout:tail.append(line.rstrip())seen["last"] = time.monotonic()- if "Pull complete" in line or "Already exists" in line:- seen["layers"] += 1+ seen["detail"] = detail(line) or seen["detail"]reader = threading.Thread(target=_read_output, daemon=True)reader.start()started = reported = time.monotonic()- _write_progress(progress_id, "pulling", "Downloading images...")+ _write_progress(progress_id, "pulling", label)failure = ""while proc.poll() is None:time.sleep(1)now = time.monotonic()if now - seen["last"] > IMAGE_PULL_STALL_SECONDS:- failure = f"Image download made no progress for {IMAGE_PULL_STALL_SECONDS // 60} minutes."+ failure = f"{subject} made no progress for {IMAGE_PULL_STALL_SECONDS // 60} minutes."elif now - started > IMAGE_PULL_MAX_SECONDS:- failure = f"Image download did not finish within {IMAGE_PULL_MAX_SECONDS // 3600} hours."+ failure = f"{subject} did not finish within {IMAGE_PULL_MAX_SECONDS // 3600} hours."if failure:proc.kill()breakif now - reported >= _IMAGE_PULL_PROGRESS_SECONDS:reported = nowminutes, seconds = divmod(int(now - started), 60)+ elapsed = f"{label} {minutes}:{seconds:02d} elapsed"_write_progress(progress_id, "pulling",- f"Downloading images... {minutes}:{seconds:02d} elapsed, "- f"{seen['layers']} layers done")+ f"{elapsed}, {seen['detail']}" if seen["detail"] else elapsed)proc.wait()reader.join(timeout=5)+ return proc.returncode, failure, list(tail)+++def _pull_compose_images(flags: list[str], service_ids: list[str], progress_id: str,+ *, env: dict[str, str] | None = None) -> tuple[bool, str]:+ """Download the images of ``service_ids`` with live progress under ``progress_id``.++ Compose resolves each image from the same files and pinned digests that+ ``up`` uses. Returns ``(ok, error)``; the pull is stopped only when Docker+ reports nothing for IMAGE_PULL_STALL_SECONDS or IMAGE_PULL_MAX_SECONDS pass.+ """+ command = ["docker", "compose", "--progress", "plain", *flags, "pull", *service_ids]+ layers = [0]++ def detail(line: str) -> str:+ if "Pull complete" in line or "Already exists" in line:+ layers[0] += 1+ return f"{layers[0]} layers done"++ returncode, failure, tail = _follow_image_command(+ command, progress_id, "Downloading images...", detail, "Image download", env=env)+ if failure:+ return False, failure+ if returncode:+ return False, "Image download failed:\n" + "\n".join(tail[-12:])+ return True, ""+++# Compose's plain build progress (Compose 5.2): "#14 [comfyui 10/15] RUN ..."+# starts a Dockerfile step, "#20 [comfyui] exporting to image" saves the image.+_BUILD_STEP_RE = re.compile(r"#\d+ \[(?:[^\]]* )?\s*(\d+)/(\d+)\]")+_BUILD_EXPORT_RE = re.compile(r"#\d+ (?:\[[^\]]*\] )?exporting to ")+++def _build_compose_images(flags: list[str], service_ids: list[str], progress_id: str,+ *, env: dict[str, str] | None = None) -> tuple[bool, str]:+ """Build the local images of ``service_ids`` with live progress under ``progress_id``.++ Compose builds from the same files ``up`` uses, so ``up`` then only+ creates the container. A first ComfyUI build for NVIDIA installs its+ Python stack and saves a 7 GB image: ten minutes on Workstation06+ (2026-10-26), all of the 600 s start allowance, so its first ``up`` timed+ out. Returns ``(ok, error)``; the build is stopped only when BuildKit+ prints nothing for IMAGE_PULL_STALL_SECONDS or IMAGE_PULL_MAX_SECONDS pass.+ """+ command = ["docker", "compose", "--progress", "plain", *flags, "build", *service_ids]++ def detail(line: str) -> str | None:+ if _BUILD_EXPORT_RE.match(line):+ return "saving the image"+ step = _BUILD_STEP_RE.match(line)+ return f"step {step[1]} of {step[2]}" if step else None++ returncode, failure, tail = _follow_image_command(+ command, progress_id, "Building images...", detail, "Image build", env=env)if failure:return False, failure- if proc.returncode:- return False, "Image download failed:\n" + "\n".join(list(tail)[-12:])+ if returncode:+ return False, _install_build_diagnostic(+ subprocess.CompletedProcess(command, returncode, "\n".join(tail), ""), {})return True, ""@@ -13465,11 +13530,13 @@ class AgentHandler(BaseHTTPRequestHandler):"""Download the images and models a planned enable needs before anything is selected.Takes service ids only: bundled extensions and Open WebUI, whose images- Compose resolves from their shipped files and pinned digests. Their- default model packs (model-packs.json) follow the images. Answers 200- when everything is already here, otherwise 202 and downloads in the- background, reporting progress under ``progress_id`` until it records- ``prepared`` or ``error``. Each service allows one operation.+ Compose resolves from their shipped files and pinned digests. Images a+ service builds from its shipped Dockerfile are built here too, so the+ start that follows only creates containers. Their default model packs+ (model-packs.json) follow the images. Answers 200 when everything is+ already here, otherwise 202 and works in the background, reporting+ progress under ``progress_id`` until it records ``prepared`` or+ ``error``. Each service allows one operation."""if not check_auth(self):return@@ -13504,32 +13571,40 @@ class AgentHandler(BaseHTTPRequestHandler):try:flags, env = _image_prepare_context(service_ids)- missing = _services_missing_images(flags, service_ids, env)+ pulls, builds = _services_missing_images(flags, service_ids, env)model_files = _missing_model_files(service_ids)except (RuntimeError, ValueError, OSError, subprocess.SubprocessError) as exc:_release()logger.warning("Image preparation could not resolve %s: %s", progress_id, type(exc).__name__)json_response(self, 503, {"error": str(exc)[:300]})return- if not missing and not model_files:+ if not pulls and not builds and not model_files:_release()json_response(self, 200, {"status": "ready", "service_ids": service_ids})returndef _download() -> None:try:- if missing:- pulled, error = _pull_compose_images(flags, missing, progress_id, env=env)+ if pulls:+ pulled, error = _pull_compose_images(flags, pulls, progress_id, env=env)if not pulled:_write_progress(progress_id, "error", "Image download failed", error=error)return+ if builds:+ built, error = _build_compose_images(flags, builds, progress_id, env=env)+ if not built:+ _write_progress(progress_id, "error", "Image build failed", error=error)+ returnif model_files:fetched, error = _fetch_model_files(progress_id, model_files)if not fetched:_write_progress(progress_id, "error", "Model download failed", error=error)return- _write_progress(progress_id, "prepared",- "Images and models downloaded" if model_files else "Images downloaded")+ if builds:+ label = "Images built and models downloaded" if model_files else "Images built"+ else:+ label = "Images and models downloaded" if model_files else "Images downloaded"+ _write_progress(progress_id, "prepared", label)except (OSError, ValueError, subprocess.SubprocessError) as exc:logger.exception("Image preparation failed for %s", progress_id)_write_progress(progress_id, "error", "Image download failed", error=str(exc)[:500])@@ -13541,8 +13616,8 @@ class AgentHandler(BaseHTTPRequestHandler):except Exception:_release()raise- json_response(self, 202, {"status": "accepted", "service_ids": service_ids, "pulling": missing,- "models": [item.target.name for item in model_files]})+ json_response(self, 202, {"status": "accepted", "service_ids": service_ids, "pulling": pulls,+ "building": builds, "models": [item.target.name for item in model_files]})# ── Model management handlers ──diff --git a/ods/docs/HOST-AGENT-API.md b/ods/docs/HOST-AGENT-API.mdindex 419ff9389..8ea7d662e 100644--- a/ods/docs/HOST-AGENT-API.md+++ b/ods/docs/HOST-AGENT-API.md@@ -290,12 +290,14 @@ Compose resolves the images from the services' shipped files and pinned digests.{"status": "ready", "service_ids": ["searxng", "hermes", "hermes-proxy"]}```-**Response (202)** while the missing images download in the background:+**Response (202)** while the missing images download or build in the background:```json-{"status": "accepted", "service_ids": ["searxng", "hermes", "hermes-proxy"], "pulling": ["hermes"]}+{"status": "accepted", "service_ids": ["searxng", "hermes", "hermes-proxy"], "pulling": ["hermes"], "building": [], "models": []}```-The progress record under `progress_id` shows `pulling`, with an elapsed time and completed layer count in `phase_label`, then `prepared` or `error`. The download stops only when Docker reports nothing for `ODS_IMAGE_PULL_STALL_SECONDS` (default 900) or after `ODS_IMAGE_PULL_MAX_SECONDS` (default 21600), not after a fixed request timeout. The Library install path downloads its images the same way.+`building` lists the services whose image Compose builds from the shipped Dockerfile (a `build:` section, such as ComfyUI on NVIDIA) and is not on the host yet, checked under the `image` name or Compose's `<project>-<service>` default. They are built after the downloads, so the start that follows only creates containers; a first ComfyUI build takes about ten minutes, longer than the 600 s start allowance.++The progress record under `progress_id` shows `pulling`, with an elapsed time and completed layer count (downloads) or the current Dockerfile step (builds) in `phase_label`, then `prepared` or `error`. A download or build stops only when Docker reports nothing for `ODS_IMAGE_PULL_STALL_SECONDS` (default 900) or after `ODS_IMAGE_PULL_MAX_SECONDS` (default 21600), not after a fixed request timeout. The Library install path downloads its images the same way.**Error responses:**| Code | Condition |diff --git a/ods/extensions/services/dashboard-api/tests/test_host_agent_image_prepare.py b/ods/extensions/services/dashboard-api/tests/test_host_agent_image_prepare.pyindex cc3d2981b..f3a89a189 100644--- a/ods/extensions/services/dashboard-api/tests/test_host_agent_image_prepare.py+++ b/ods/extensions/services/dashboard-api/tests/test_host_agent_image_prepare.py@@ -1,4 +1,4 @@-"""Image downloads that outlast a request: prepare-images and the streaming pull.+"""Image work that outlasts a request: prepare-images, the streaming pull and build.On the Mac mini's slow link (2026-10-04) Hermes' first `docker compose up`spent its whole 600 s start allowance downloading, and adding Open WebUI died@@ -6,6 +6,12 @@ at 600 s with the 0.11.4 image unfinished, while the Dashboard had given up at180 s. Bundled services now download their images first through/v1/extension/prepare-images, before anything is selected, with progress onthe card; the pull stops only when Docker stops making progress.++On Workstation06 (2026-10-26) ComfyUI's first NVIDIA image build took the+whole 600 s start allowance (ten minutes, 0% cached), so the start timed out+and the card showed ComfyUI stopped. Images a bundled service builds from its+shipped Dockerfile are now built in the same step, with the build step on the+card."""import json@@ -116,12 +122,93 @@ def test_failed_pull_returns_dockers_last_words(host, monkeypatch, clock): # noassert error.splitlines() == ["Image download failed:", " Image x Pulling", "toomanyrequests: rate limit"]-def test_missing_images_skips_local_builds_and_names_the_services(host, monkeypatch): # noqa: F811- config = {"services": {+class TimedBuild(FakePull):+ """`docker compose build`: prints its lines at once, then runs until the faked clock reaches ``exit_at``."""++ def __init__(self, command, clock, exit_at, lines=(), returncode=0):+ super().__init__(command, lines=lines, returncode=returncode)+ self._clock = clock+ self._exit_at = exit_at++ def poll(self):+ if self._clock[0] < self._exit_at and not self.killed:+ return None+ return super().poll()++[email protected]("lines, detail", [+ (["#5 [comfyui 1/15] FROM docker.io/nvidia/cuda:12.8.0-runtime-ubuntu22.04\n",+ "#14 [comfyui 10/15] RUN pip3 install --no-cache-dir comfy-kitchen\n",+ "#14 12.31 Collecting comfy-kitchen\n"], "step 10 of 15"),+ # Compose 5.2 names the target in the export line; its sub-lines do not.+ (["#19 [comfyui 15/15] WORKDIR /opt/comfyui\n", "#20 [comfyui] exporting to image\n",+ "#20 exporting layers 125.3s done\n"], "saving the image"),+])+def test_build_reports_its_step_and_uses_the_compose_files_up_uses(host, monkeypatch, clock, lines, detail): # noqa: F811+ started = []++ def popen(command, **options):+ started.append((list(command), options))+ return TimedBuild(command, clock, clock[0] + 40, lines=lines)++ monkeypatch.setattr(_mod.subprocess, "Popen", popen)+ written = []+ monkeypatch.setattr(_mod, "_write_progress", lambda *args, **kwargs: written.append(args))++ ok, error = _mod._build_compose_images(["-f", "base.yml"], ["comfyui"], "comfyui")++ assert (ok, error) == (True, "")+ command, options = started[0]+ assert command == ["docker", "compose", "--progress", "plain", "-f", "base.yml", "build", "comfyui"]+ assert options["stderr"] is subprocess.STDOUT+ labels = [args[2] for args in written if args[:2] == ("comfyui", "pulling")]+ assert labels[0] == "Building images..."+ assert f"Building images... 0:15 elapsed, {detail}" in labels+++def test_build_stops_when_buildkit_reports_nothing(host, monkeypatch, clock): # noqa: F811+ monkeypatch.setattr(_mod, "IMAGE_PULL_STALL_SECONDS", 120)+ monkeypatch.setattr(_mod, "IMAGE_PULL_MAX_SECONDS", 10 ** 6)+ build = FakePull(["cmd"], hang=True)+ monkeypatch.setattr(_mod.subprocess, "Popen", lambda command, **options: build)+ written = []+ monkeypatch.setattr(_mod, "_write_progress", lambda *args, **kwargs: written.append(args))++ ok, error = _mod._build_compose_images([], ["comfyui"], "comfyui")++ assert ok is False and build.killed+ assert error == "Image build made no progress for 2 minutes."+ # Before the first Dockerfile step there is no step to name.+ assert ("comfyui", "pulling", "Building images... 0:15 elapsed") in written+++def test_failed_build_leads_with_buildkits_decisive_error_without_credentials(host, monkeypatch, clock): # noqa: F811+ _popen(monkeypatch, returncode=1, lines=[+ "#14 [comfyui 10/15] RUN pip3 install comfy-kitchen\n",+ "#14 0.52 token SERVICE_API_KEY=persisted-credential\n",+ "#14 ERROR: process did not complete successfully: exit code: 1\n",+ 'failed to solve: process "/bin/sh -c pip3 install comfy-kitchen" did not complete successfully: exit code: 1\n',+ ])+ monkeypatch.setattr(_mod, "_write_progress", lambda *args, **kwargs: None)++ ok, error = _mod._build_compose_images([], ["comfyui"], "comfyui")++ assert ok is False+ assert error.startswith('Untrusted build error: failed to solve: process "/bin/sh -c pip3 install')+ assert "#14 [comfyui 10/15] RUN pip3 install comfy-kitchen" in error+ assert "persisted-credential" not in error+++def test_missing_images_names_pulls_and_local_builds_apart(host, monkeypatch): # noqa: F811+ config = {"name": "ods", "services": {"hermes": {"image": "nousresearch/hermes-agent@sha256:" + "a" * 64},"hermes-proxy": {"image": "caddy:2"},"builder": {"image": "ods-built:local", "build": {"context": "."}},+ # No image: Compose names what `up` builds <project>-<service>.+ "comfyui": {"build": {"context": "./extensions/services/comfyui"}},+ "token-spy": {"build": {"context": "./extensions/services/token-spy"}},}}+ present = {"caddy:2", "ods-token-spy"}seen = []def run(command, **kwargs):@@ -130,20 +217,26 @@ def test_missing_images_skips_local_builds_and_names_the_services(host, monkeypaassert kwargs["env"] == {"ENABLE_OPEN_WEBUI": "true"}return types.SimpleNamespace(returncode=0, stdout=json.dumps(config), stderr="")if command[:3] == ["docker", "image", "inspect"]:- return types.SimpleNamespace(returncode=0 if command[3] == "caddy:2" else 1, stdout="", stderr="")+ return types.SimpleNamespace(returncode=0 if command[3] in present else 1, stdout="", stderr="")raise AssertionError(command)monkeypatch.setattr(_mod.subprocess, "run", run)- missing = _mod._services_missing_images(["-f", "b.yml"], ["hermes", "hermes-proxy", "builder"],- {"ENABLE_OPEN_WEBUI": "true"})- assert missing == ["hermes"]- assert ["docker", "image", "inspect", "ods-built:local"] not in seen+ pulls, builds = _mod._services_missing_images(+ ["-f", "b.yml"], ["hermes", "hermes-proxy", "builder", "comfyui", "token-spy"],+ {"ENABLE_OPEN_WEBUI": "true"})+ assert (pulls, builds) == (["hermes"], ["builder", "comfyui"])+ assert ["docker", "image", "inspect", "ods-comfyui"] in seen-def test_missing_images_refuses_a_service_the_stack_does_not_define(host, monkeypatch): # noqa: F811[email protected]("config, message", [+ ({"name": "ods", "services": {}}, "does not define hermes"),+ # Without the project name a build-only service's image cannot be named.+ ({"services": {"hermes": {"build": {"context": "."}}}}, "Invalid Compose configuration"),+])+def test_missing_images_refuses_a_configuration_it_cannot_read(host, monkeypatch, config, message): # noqa: F811monkeypatch.setattr(_mod.subprocess, "run", lambda command, **kwargs: types.SimpleNamespace(- returncode=0, stdout=json.dumps({"services": {}}), stderr=""))- with pytest.raises(RuntimeError, match="does not define hermes"):+ returncode=0, stdout=json.dumps(config), stderr=""))+ with pytest.raises(RuntimeError, match=message):_mod._services_missing_images([], ["hermes"], {})@@ -212,7 +305,7 @@ def test_prepare_refuses_library_and_unknown_services(prepare, host): # noqa: Fdef test_prepare_answers_ready_without_downloading_when_images_are_here(prepare, host, monkeypatch): # noqa: F811- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))monkeypatch.setattr(_mod, "_pull_compose_images", lambda *args, **kwargs: pytest.fail("no download expected"))status, response = prepare({"service_ids": ["hermes", "hermes-proxy"], "progress_id": "hermes"})@@ -222,7 +315,7 @@ def test_prepare_answers_ready_without_downloading_when_images_are_here(prepare,def test_prepare_downloads_in_the_background_and_records_the_result(prepare, host, monkeypatch): # noqa: F811- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ["hermes"])+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: (["hermes"], []))pulls = []def pull(flags, services, progress_id, *, env=None):@@ -242,7 +335,7 @@ def test_prepare_downloads_in_the_background_and_records_the_result(prepare, hosdef test_prepare_records_a_failed_download_for_the_card(prepare, host, monkeypatch): # noqa: F811- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ["open-webui"])+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: (["open-webui"], []))monkeypatch.setattr(_mod, "_pull_compose_images",lambda *args, **kwargs: (False, "Image download made no progress for 15 minutes."))@@ -255,6 +348,22 @@ def test_prepare_records_a_failed_download_for_the_card(prepare, host, monkeypatassert not _mod._service_locks["open-webui"].locked()+def test_prepare_records_a_failed_build_for_the_card(prepare, host, monkeypatch): # noqa: F811+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], ["hermes"]))+ monkeypatch.setattr(_mod, "_pull_compose_images", lambda *args, **kwargs: pytest.fail("nothing to download"))+ monkeypatch.setattr(_mod, "_build_compose_images",+ lambda *args, **kwargs: (False, "Untrusted build error: failed to solve"))++ status, response = prepare({"service_ids": ["hermes"], "progress_id": "hermes"})++ # A missing local image alone is not ready: `up` would build it inside the start.+ assert status == 202 and (response["pulling"], response["building"]) == ([], ["hermes"])+ record = host.progress("hermes")+ assert (record["status"], record["phase_label"]) == ("error", "Image build failed")+ assert record["error"] == "Untrusted build error: failed to solve"+ assert not _mod._service_locks["hermes"].locked()++def test_prepare_refuses_while_the_service_is_being_changed(prepare, host, monkeypatch): # noqa: F811monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: pytest.fail("not reached"))_mod._service_locks["hermes-proxy"].acquire()@@ -291,7 +400,7 @@ def model_pack(prepare, host, monkeypatch): # noqa: F811def test_prepare_fetches_a_missing_model_pack_after_the_images(prepare, host, monkeypatch, model_pack): # noqa: F811- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ["painter"])+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: (["painter"], []))order = []def pull(*args, **kwargs):@@ -299,6 +408,7 @@ def test_prepare_fetches_a_missing_model_pack_after_the_images(prepare, host, moreturn True, ""monkeypatch.setattr(_mod, "_pull_compose_images", pull)+ monkeypatch.setattr(_mod, "_build_compose_images", lambda *args, **kwargs: pytest.fail("nothing to build"))def fetch(item, report):order.append("model")@@ -319,8 +429,42 @@ def test_prepare_fetches_a_missing_model_pack_after_the_images(prepare, host, moassert not _mod._service_locks["painter"].locked()+def test_prepare_builds_local_images_after_downloads_and_before_models(prepare, host, monkeypatch, model_pack): # noqa: F811+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: (["hermes"], ["painter"]))+ order = []++ def pull(flags, services, progress_id, *, env=None):+ order.append(("pull", services))+ return True, ""++ def build(flags, services, progress_id, *, env=None):+ order.append(("build", services))+ # The files and selectors the enable will run, under the card's record.+ assert (flags, progress_id, env) == (["-f", "base.yml"], "painter", {"E": "1"})+ assert _mod._service_locks["painter"].locked() # one operation per service+ return True, ""++ def fetch(item, report):+ order.append(("model", item.target.name))+ item.target.parent.mkdir(parents=True)+ item.target.write_bytes(b"model")++ monkeypatch.setattr(_mod, "_pull_compose_images", pull)+ monkeypatch.setattr(_mod, "_build_compose_images", build)+ monkeypatch.setattr(_mod._extension_assets, "fetch", fetch)++ status, response = prepare({"service_ids": ["hermes", "painter"], "progress_id": "painter"})++ assert status == 202+ assert (response["pulling"], response["building"]) == (["hermes"], ["painter"])+ assert order == [("pull", ["hermes"]), ("build", ["painter"]), ("model", PACK_FILE)]+ record = host.progress("painter")+ assert (record["status"], record["phase_label"]) == ("prepared", "Images built and models downloaded")+ assert not _mod._service_locks["painter"].locked()++def test_prepare_downloads_a_missing_model_even_when_the_images_are_here(prepare, host, monkeypatch, model_pack): # noqa: F811- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))monkeypatch.setattr(_mod, "_pull_compose_images", lambda *args, **kwargs: pytest.fail("no image download"))labels = []real_write = _mod._write_progress@@ -343,7 +487,7 @@ def test_prepare_downloads_a_missing_model_even_when_the_images_are_here(preparedef test_prepare_is_ready_when_images_and_models_are_here(prepare, host, monkeypatch, model_pack): # noqa: F811model_pack.parent.mkdir(parents=True)model_pack.write_bytes(b"model") # the pinned size: 5 bytes- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))monkeypatch.setattr(_mod._extension_assets, "fetch", lambda *args: pytest.fail("no download expected"))status, response = prepare({"service_ids": ["painter"], "progress_id": "painter"})@@ -353,7 +497,7 @@ def test_prepare_is_ready_when_images_and_models_are_here(prepare, host, monkeypdef test_prepare_records_a_failed_model_download_for_the_card(prepare, host, monkeypatch, model_pack): # noqa: F811- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))def fail(item, report):raise _mod._extension_assets.AssetError("The download of Painter model stopped (timed out).")@@ -371,7 +515,7 @@ def test_prepare_records_a_failed_model_download_for_the_card(prepare, host, mondef test_prepare_reports_a_malformed_pack_list_without_downloading(prepare, host, monkeypatch, model_pack): # noqa: F811(host.builtins / "painter" / "model-packs.json").write_text('{"schema": "other"}', encoding="utf-8")- monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: [])+ monkeypatch.setattr(_mod, "_services_missing_images", lambda flags, ids, env: ([], []))status, response = prepare({"service_ids": ["painter"], "progress_id": "painter"})