Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(models): reject unsupported architectures before activation (#8225)

Mike Bradley373b322938f4
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.py
index 0b26764fa..ef280447a 100755
--- a/ods/bin/ods-host-agent.py
+++ b/ods/bin/ods-host-agent.py
@@ -134,6 +134,54 @@ if _model_stores_spec is None or _model_stores_spec.loader is None:
_model_stores = importlib.util.module_from_spec(_model_stores_spec)
_model_stores_spec.loader.exec_module(_model_stores)
+_model_preflight_spec = importlib.util.spec_from_file_location(
+ "_ods_model_preflight", _MODEL_MEMORY_PATH.with_name("model_preflight.py"))
+if _model_preflight_spec is None or _model_preflight_spec.loader is None:
+ raise ImportError("Cannot load shared runtime compatibility policy")
+_model_preflight = importlib.util.module_from_spec(_model_preflight_spec)
+_model_preflight_spec.loader.exec_module(_model_preflight)
+
+_gguf_inspector_spec = importlib.util.spec_from_file_location(
+ "_ods_gguf_inspector", _MODEL_MEMORY_PATH.with_name("gguf_inspector.py"))
+if _gguf_inspector_spec is None or _gguf_inspector_spec.loader is None:
+ raise ImportError("Cannot load shared GGUF inspector")
+_gguf_inspector = importlib.util.module_from_spec(_gguf_inspector_spec)
+_gguf_inspector_spec.loader.exec_module(_gguf_inspector)
+
+
+def _activation_architecture_refusal(target: Path, env: dict, *, windows_hosted: bool,
+ registered_runtime: dict | None) -> dict | None:
+ """Refuse a known unsupported architecture before holding or changing a route.
+
+ Import acknowledgements do not qualify the installed runtime. Inspect the
+ actual selected file again, including files added directly to a model store.
+ Custom runtimes have their own qualification; the bundled policy cannot
+ establish their capabilities. Unknown evidence retains the existing guarded
+ activation and rollback rather than inventing an incompatibility.
+ """
+ if registered_runtime:
+ return None
+ header = _gguf_inspector.inspect_gguf(target)
+ architecture = header.get("architecture")
+ if not header.get("readable") or architecture in {None, "", "unknown"}:
+ return None
+ backend = str(env.get("GPU_BACKEND") or "nvidia").strip().lower()
+ image = str(env.get("LLAMA_SERVER_IMAGE") or "").strip() if backend not in {"apple", "amd"} and not windows_hosted else ""
+ if image and (not image.startswith(f"{_LLAMA_CPP_REPOSITORY}:server-")
+ or _model_preflight.build_from_image(image) is None):
+ return None
+ policy = _model_preflight.load_runtime_policy(INSTALL_DIR / "config/llama-cpp-architectures.json")
+ key, build = _model_preflight.effective_build(policy, backend, windows_hosted=windows_hosted, model_image=image)
+ if _model_preflight.architecture_supported(policy, build, architecture) is not False:
+ return None
+ return {
+ "code": "runtime_architecture_unsupported",
+ "error": (f"Model architecture '{architecture}' is not supported by the selected llama.cpp build {build}. "
+ "Choose a compatible model or qualify a runtime that supports it. The active model was not changed."),
+ "architecture": architecture,
+ "runtime": {"key": key, "build": build},
+ }
+
def _installed_model_file(filename: str) -> Path | None:
return _model_stores.resolve_model_file(INSTALL_DIR / "data", filename, container=bool(os.environ.get("ODS_HOST_INSTALL_DIR")))
@@ -15114,6 +15162,15 @@ class AgentHandler(BaseHTTPRequestHandler):
profile_context_length = None
llama_server_image = runtime_profile.get("llama_server_image") or llama_server_image
runtime_env = runtime_profile.get("env") if isinstance(runtime_profile.get("env"), dict) else {}
+ image_updates, image_removals = _model_activation_image_changes(env_pre, llama_server_image, catalog_images)
+ selected_runtime_env = {key: value for key, value in env_pre.items() if key not in image_removals}
+ selected_runtime_env.update(image_updates)
+ architecture_refusal = _activation_architecture_refusal(
+ target, selected_runtime_env, windows_hosted=host_native_llama,
+ registered_runtime=local_runtime_profile)
+ if architecture_refusal is not None:
+ json_response(self, 409, {**architecture_refusal, "requestedModelId": model_id})
+ return
recommended_context = _recommended_activation_context(model_id, model, env_pre)
if (
requested_context_length is None
@@ -15319,34 +15376,10 @@ class AgentHandler(BaseHTTPRequestHandler):
"LLAMA_ARG_SPEC_DRAFT_TYPE_K":"q4_0", "LLAMA_ARG_SPEC_DRAFT_TYPE_V":"q4_0"})
remove_keys.update({"LLAMA_ARG_SPEC_DRAFT_TYPE_K", "LLAMA_ARG_SPEC_DRAFT_TYPE_V"})
remove_keys.difference_update(updates)
- # Only update LLAMA_SERVER_IMAGE on Docker backends that read it.
- # macOS runs llama-server natively (no Docker image to pull),
- # and the AMD overlay pins its own Vulkan/ROCm image: catalog
- # images name another backend's (CUDA) build.
- if llama_server_image and gpu_backend not in {"apple", "amd"}:
- updates["LLAMA_SERVER_IMAGE"] = llama_server_image
- elif (
- gpu_backend not in {"apple", "amd"}
- and str(env_pre.get("LLAMA_SERVER_IMAGE") or "").strip() in catalog_images
- ):
- # A catalog entry's or runtime profile's image belongs to
- # the model that selected it. A model without its own image
- # runs the backend default again; a host or owner image that
- # no catalog model names is kept.
- remove_keys.add("LLAMA_SERVER_IMAGE")
- if gpu_backend not in {"apple", "amd"}:
- # A compute-12.1 GPU names its CUDA 13 build here: the
- # backend default (no key) is a CUDA 12 build.
- if "LLAMA_SERVER_IMAGE" in updates:
- env_image = updates["LLAMA_SERVER_IMAGE"]
- elif "LLAMA_SERVER_IMAGE" in remove_keys:
- env_image = ""
- else:
- env_image = str(env_pre.get("LLAMA_SERVER_IMAGE") or "").strip()
- compute_image = _llama_image_for_compute(env_image, env_pre)
- if compute_image != env_image:
- updates["LLAMA_SERVER_IMAGE"] = compute_image
- remove_keys.discard("LLAMA_SERVER_IMAGE")
+ # Apply exactly the runtime selection checked before mutation.
+ updates.update(image_updates)
+ remove_keys.update(image_removals)
+ remove_keys.difference_update(image_updates)
# A vision import loads its projector in the container (WP2);
# any other model clears the previous one's. Host-native
# runtimes take the projector as an argument instead.
@@ -17802,6 +17835,27 @@ def _render_model_router_runtime_configs(
logger.warning("%s; switchboard mode is %s", message, switchboard_mode)
+def _model_activation_image_changes(env: dict, model_image: str | None,
+ catalog_images: set[str] | frozenset[str]) -> tuple[dict, set]:
+ """Select the candidate image once for both compatibility and activation."""
+ updates: dict = {}
+ removals: set = set()
+ if env.get("GPU_BACKEND", "nvidia") in {"apple", "amd"}:
+ return updates, removals # Native macOS and the AMD overlay ignore this key.
+ current = str(env.get("LLAMA_SERVER_IMAGE") or "").strip()
+ if model_image:
+ updates["LLAMA_SERVER_IMAGE"] = model_image
+ elif current in catalog_images:
+ # A previous model's image is not an owner override.
+ removals.add("LLAMA_SERVER_IMAGE")
+ image = updates.get("LLAMA_SERVER_IMAGE", "" if removals else current)
+ compute_image = _llama_image_for_compute(image, env)
+ if compute_image != image:
+ updates["LLAMA_SERVER_IMAGE"] = compute_image
+ removals.discard("LLAMA_SERVER_IMAGE")
+ return updates, removals
+
+
def _container_llama_rollback_image(env: dict) -> str:
"""Image to recreate a rolled-back container with ("" keeps the inspected).
diff --git a/ods/extensions/services/dashboard-api/tests/test_model_activate.py b/ods/extensions/services/dashboard-api/tests/test_model_activate.py
index 416373968..259f0af72 100644
--- a/ods/extensions/services/dashboard-api/tests/test_model_activate.py
+++ b/ods/extensions/services/dashboard-api/tests/test_model_activate.py
@@ -7,6 +7,7 @@ import http.client
import io
import json
import shutil
+import struct
import subprocess
import sys
import threading
@@ -3913,6 +3914,80 @@ def test_pixel_reasoning_capability_follows_model_family_and_runtime_mode(
) is expected
[email protected]('backend', ['nvidia', 'cpu', 'amd', 'apple'])
[email protected]('catalog', [True, False])
+def test_unsupported_architecture_preserves_running_model_before_transaction(tmp_path, monkeypatch, backend, catalog):
+ install, env_path, env_text, models_ini, ini_text, *_ = _write_model_activation_fixture(tmp_path, gpu_backend=backend)
+ def string(value):
+ raw = value.encode()
+ return struct.pack('<Q', len(raw)) + raw
+ data = (b'GGUF' + struct.pack('<IQQ', 3, 0, 1)
+ + string('general.architecture') + struct.pack('<I', 8) + string('dspark'))
+ (install / 'data/models/new-model.gguf').write_bytes(data)
+ library = {'models': [{'id': 'target-model', 'gguf_file': 'new-model.gguf',
+ 'gguf_url': 'https://example.test/new-model.gguf',
+ 'gguf_sha256': hashlib.sha256(data).hexdigest(),
+ 'llm_model_name': 'new-model', 'context_length': 4096}]} if catalog else {'models': []}
+ (install / 'config/model-library.json').write_text(json.dumps(library))
+ shutil.copyfile(_agent_path.parents[1] / 'config/llama-cpp-architectures.json', install / 'config/llama-cpp-architectures.json')
+ monkeypatch.setattr(_mod, 'INSTALL_DIR', install)
+ monkeypatch.setattr(_mod.platform, 'system', lambda: 'Linux')
+ monkeypatch.delenv('ODS_HOST_INSTALL_DIR', raising=False)
+ monkeypatch.setattr(_mod, '_ods_managed_pixel_identity', lambda: None)
+ monkeypatch.setattr(_mod, '_recommended_activation_context', lambda *_a: None)
+ monkeypatch.setattr(_mod, '_select_runtime_profile', lambda *_a: None)
+ monkeypatch.setattr(_mod, '_managed_wsl_runtime', lambda *_a: {'managed': False})
+ monkeypatch.setattr(_mod, '_plan_nvidia_model_gpu_assignment', lambda *_a, **_kw: None)
+ monkeypatch.setattr(_mod, '_plan_amd_model_gpu_assignment', lambda *_a, **_kw: None)
+ reached = []
+ def snapshot(*_args):
+ reached.append('snapshot')
+ raise RuntimeError('test stopped before any transaction or process mutation')
+ monkeypatch.setattr(_mod, '_snapshot_text_file', snapshot)
+ handler = _ResponseHandler()
+ _mod.AgentHandler._do_model_activate(handler, 'target-model' if catalog else 'new-model')
+ assert handler.response_code == 409, handler.parse_response()
+ assert handler.parse_response()['code'] == 'runtime_architecture_unsupported'
+ assert handler.parse_response()['architecture'] == 'dspark'
+ assert reached == []
+ assert env_path.read_text() == env_text
+ assert models_ini.read_text() == ini_text
+ assert not (install / 'data/model-activation-receipt.json').exists()
+
+
[email protected]('architecture,image,registered,policy_present,windows,blocked,build', [
+ ('qwen35', '', None, True, False, False, None),
+ ('dspark', '', None, True, False, True, 'b11429'),
+ ('dspark', 'ghcr.io/ggml-org/llama.cpp:server-cuda-b9014@sha256:example', None, True, False, True, 'b9014'),
+ ('dspark', 'ghcr.io/ggml-org/llama.cpp:server-cuda-b99999', None, True, False, False, None),
+ ('dspark', 'ghcr.io/ggml-org/llama.cpp:server-cuda-custom', None, True, False, False, None),
+ ('dspark', 'owner/custom:server-b9014', None, True, False, False, None),
+ ('dspark', '', {'executable': 'qualified-custom-runtime'}, True, False, False, None),
+ ('dspark', '', None, False, False, False, None),
+ ('unknown', '', None, True, False, False, None),
+ ('dspark', 'owner/ignored-image:latest', None, True, True, True, 'b9014'),
+])
+def test_activation_architecture_uses_selected_runtime_and_positive_evidence(
+ tmp_path, monkeypatch, architecture, image, registered, policy_present, windows, blocked, build):
+ (tmp_path / 'config').mkdir()
+ if policy_present:
+ shutil.copyfile(_agent_path.parents[1] / 'config/llama-cpp-architectures.json', tmp_path / 'config/llama-cpp-architectures.json')
+ def string(value):
+ data = value.encode()
+ return struct.pack('<Q', len(data)) + data
+ target = tmp_path / 'test.gguf'
+ target.write_bytes(b'GGUF' + struct.pack('<IQQ', 3, 0, 1)
+ + string('general.architecture') + struct.pack('<I', 8) + string(architecture))
+ monkeypatch.setattr(_mod, 'INSTALL_DIR', tmp_path)
+ refusal = _mod._activation_architecture_refusal(
+ target, {'GPU_BACKEND': 'nvidia', 'LLAMA_SERVER_IMAGE': image},
+ windows_hosted=windows, registered_runtime=registered)
+ assert (refusal is not None) is blocked
+ if blocked:
+ assert refusal['architecture'] == architecture
+ assert refusal['runtime']['build'] == build
+
+
class TestModelActivateRollback:
@pytest.fixture(autouse=True)
diff --git a/ods/extensions/services/dashboard/src/hooks/useModels.js b/ods/extensions/services/dashboard/src/hooks/useModels.js
index fb334b655..8cb627185 100644
--- a/ods/extensions/services/dashboard/src/hooks/useModels.js
+++ b/ods/extensions/services/dashboard/src/hooks/useModels.js
@@ -592,6 +592,10 @@ export function useModels({observe=true, reuseLastCatalog=false} = {}) {
const body = await responseJson(response)
if (response.status === 409) {
+ if (body?.detail?.code === 'runtime_architecture_unsupported') {
+ activationError = errorMessageFromPayload(body, 'This runtime does not support the model architecture.')
+ return
+ }
if (body?.detail?.code === 'pixel_chat_active') {
activationError = errorMessageFromPayload(
body,
diff --git a/ods/extensions/services/dashboard/src/hooks/useModels.test.jsx b/ods/extensions/services/dashboard/src/hooks/useModels.test.jsx
index d8962dabe..54e2d86ce 100644
--- a/ods/extensions/services/dashboard/src/hooks/useModels.test.jsx
+++ b/ods/extensions/services/dashboard/src/hooks/useModels.test.jsx
@@ -71,6 +71,20 @@ test('a joined activation that loads the model is confirmed', async () => {
expect(view.result.current.currentModel).toBe('model-b')
})
+test('an unsupported architecture shows the refusal and keeps the current model without joining an activation', async () => {
+ const message = "Model architecture 'dspark' is not supported by this runtime. The active model was not changed."
+ stubServer(() => reply(409, { detail: {
+ code: 'runtime_architecture_unsupported', error: message, requestedModelId: 'model-b',
+ } }))
+ server.lifecycle = null
+ const { view, load } = await startLoad()
+ await act(async () => { await load })
+ expect(view.result.current.error).toBe(message)
+ expect(view.result.current.currentModel).toBe('model-a')
+ expect(view.result.current.activationLoading).toBeNull()
+ expect(fetch.mock.calls.filter(([url]) => url.endsWith('/load'))).toHaveLength(1)
+})
+
test('a load request still in flight keeps waiting for its answer', async () => {
stubServer(() => new Promise(() => {}))
const { view } = await startLoad()