Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(models): retain downloads after inconclusive integrity checks (#8230)

Mike Bradley7851f019d2ce
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.py
index ef280447a..bddbee34b 100755
--- a/ods/bin/ods-host-agent.py
+++ b/ods/bin/ods-host-agent.py
@@ -14136,10 +14136,14 @@ class AgentHandler(BaseHTTPRequestHandler):
else f"{part_file_name} (part {part_idx}/{len(download_plan)})"
)
_write_model_status(status_path, "verifying", verify_label, final_size, final_size)
+ # An inconclusive inspection must leave downloaded bytes
+ # available for a verified retry, just like existing files.
+ # Definitive size/checksum failures still return False.
valid, reason = _verify_model_artifact(
final_target,
artifact,
_model_download_cancel,
+ raise_on_error=True,
)
if _model_download_cancel.is_set():
_finish_cancelled_download()
diff --git a/ods/extensions/services/dashboard-api/tests/test_host_agent.py b/ods/extensions/services/dashboard-api/tests/test_host_agent.py
index 3cc32b71f..61836c79e 100644
--- a/ods/extensions/services/dashboard-api/tests/test_host_agent.py
+++ b/ods/extensions/services/dashboard-api/tests/test_host_agent.py
@@ -9942,6 +9942,102 @@ class TestModelDeleteSafety:
class TestModelDownloadFileIntegrity:
+ @pytest.mark.parametrize("failure", ["metadata-change", "read-error"])
+ def test_inconclusive_final_verification_retains_bytes_for_verified_retry(
+ self, tmp_path, monkeypatch, failure,
+ ):
+ payload = b"valid gguf bytes"
+ install_dir = self._setup_env(tmp_path, monkeypatch, expected_payload=payload)
+ target = install_dir / "data/models/test-model.gguf"
+ calls = self._patch_curl_download(monkeypatch, payload)
+ real_sha256 = hashlib.sha256
+ entered: list[bool] = []
+
+ class InterruptedDigest:
+ def __init__(self, *args, **kwargs):
+ self.digest = real_sha256(*args, **kwargs)
+
+ def update(self, chunk):
+ self.digest.update(chunk)
+ if entered:
+ return
+ entered.append(True)
+ if failure == "read-error":
+ raise OSError("Transient model read failure")
+ before = target.stat()
+ os.utime(target, ns=(before.st_atime_ns, before.st_mtime_ns + 1_000_000_000))
+
+ def hexdigest(self):
+ return self.digest.hexdigest()
+
+ monkeypatch.setattr(_mod.hashlib, "sha256", InterruptedDigest)
+ handler = _FakeHandler(self._body())
+ _mod.AgentHandler._handle_model_download(handler)
+ assert handler.response_code == 200
+ _mod._model_download_thread.join(timeout=3)
+ assert not _mod._model_download_thread.is_alive()
+ assert entered == [True]
+ status_path = install_dir / "data/model-download-status.json"
+ status = json.loads(status_path.read_text())
+ assert status["status"] == "failed"
+ expected = "file changed during verification" if failure == "metadata-change" else "Transient model read failure"
+ assert expected in status["error"]
+ assert target.read_bytes() == payload
+ assert calls == ["test-model.gguf.part"]
+
+ monkeypatch.setattr(_mod.hashlib, "sha256", real_sha256)
+ monkeypatch.setattr(_mod.subprocess, "Popen", lambda *a, **kw: pytest.fail("retry must reuse verified bytes"))
+ retry = _FakeHandler(self._body())
+ _mod.AgentHandler._handle_model_download(retry)
+ assert retry.response_code == 200
+ _mod._model_download_thread.join(timeout=3)
+ assert not _mod._model_download_thread.is_alive()
+ assert json.loads(status_path.read_text())["status"] == "complete"
+ assert target.read_bytes() == payload
+
+ def test_final_checksum_mismatch_still_removes_downloaded_bytes(self, tmp_path, monkeypatch):
+ payload = b"valid gguf bytes"
+ install_dir = self._setup_env(tmp_path, monkeypatch, expected_payload=payload)
+ self._patch_curl_download(monkeypatch, b"other gguf bytes")
+ assert len(payload) == len(b"other gguf bytes")
+ handler = _FakeHandler(self._body())
+ _mod.AgentHandler._handle_model_download(handler)
+ _mod._model_download_thread.join(timeout=3)
+ assert not _mod._model_download_thread.is_alive()
+ status = json.loads((install_dir / "data/model-download-status.json").read_text())
+ assert status["status"] == "failed"
+ assert "SHA256 mismatch" in status["error"]
+ assert not (install_dir / "data/models/test-model.gguf").exists()
+
+ def test_cancel_final_verification_still_removes_downloaded_bytes(self, tmp_path, monkeypatch):
+ payload = b"valid gguf bytes"
+ install_dir = self._setup_env(tmp_path, monkeypatch, expected_payload=payload)
+ self._patch_curl_download(monkeypatch, payload)
+ cancel = threading.Event()
+ monkeypatch.setattr(_mod, "_model_download_cancel", cancel)
+ real_sha256 = hashlib.sha256
+
+ class CancelDigest:
+ def __init__(self, *args, **kwargs):
+ self.digest = real_sha256(*args, **kwargs)
+
+ def update(self, chunk):
+ self.digest.update(chunk)
+ cancel.set()
+
+ def hexdigest(self):
+ return self.digest.hexdigest()
+
+ monkeypatch.setattr(_mod.hashlib, "sha256", CancelDigest)
+ handler = _FakeHandler(self._body())
+ _mod.AgentHandler._handle_model_download(handler)
+ _mod._model_download_thread.join(timeout=3)
+ assert not _mod._model_download_thread.is_alive()
+ status = json.loads((install_dir / "data/model-download-status.json").read_text())
+ assert status["status"] == "cancelled"
+ assert not (install_dir / "data/models/test-model.gguf").exists()
+ assert _mod._model_download_cancelable is False
+
@pytest.mark.parametrize("sized", [False, True])
def test_corrupt_existing_file_checks_space_before_any_network(self, tmp_path, monkeypatch, sized):
payload = b"valid gguf bytes"