Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

34.7 KB
Raw Blame
"""Model Switchboard PR 1 tests: state module, API endpoint, observe hook."""

from __future__ import annotations

import json
import os
import stat
import sys
import threading
import time
from pathlib import Path

import pytest

_BIN_DIR = Path(__file__).resolve().parents[4] / "bin"
if str(_BIN_DIR) not in sys.path:
    sys.path.insert(0, str(_BIN_DIR))

from model_switchboard import state as sb  # noqa: E402

_SCHEMA_PATH = Path(__file__).resolve().parents[4] / "config" / "model-state.schema.v1.json"


def _record(path, model="qwen3.5-9b", runtime="Qwen3.5-9B-Q4_K_M.gguf", backend="llama-server",
            endpoint="llama-server-default", context=32768):
    return sb.record_verified_route(
        path,
        catalog_id=model,
        runtime_model_id=runtime,
        backend_kind=backend,
        endpoint_id=endpoint,
        context_length=context,
        capabilities={"chat": True, "tools": False, "vision": False, "agentViable": context >= 65536},
        proof_identity=runtime,
    )


class TestStateModule:
    def test_roundtrip_and_schema_agreement(self, tmp_path):
        path = tmp_path / "model-state.json"
        doc = _record(path)
        assert doc["seq"] == 1 and doc["routeSeq"] == 1
        read, errors = sb.read_state(path)
        assert errors == [] and read == doc
        assert read["active"]["runtimeModelId"] == "Qwen3.5-9B-Q4_K_M.gguf"
        assert read["active"]["proof"] == {"identity": "Qwen3.5-9B-Q4_K_M.gguf", "completion": True}
        assert sb.validate_state(read) == []
        jsonschema = pytest.importorskip("jsonschema")
        schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))
        jsonschema.validate(read, schema)

    def test_vllm_route_roundtrips_and_matches_schema(self, tmp_path):
        path = tmp_path / "model-state.json"
        doc = _record(path, model="glm-5.3-flash-nvfp4", runtime="GLM-5.3-Flash-NVFP4",
                      backend="vllm", endpoint="vllm-default", context=131072)
        assert doc["active"]["backend"] == {
            "kind": "vllm", "endpointId": "vllm-default", "nativeRoute": None,
        }
        read, errors = sb.read_state(path)
        assert errors == [] and read == doc
        assert sb.validate_state(read) == []
        jsonschema = pytest.importorskip("jsonschema")
        schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))
        jsonschema.validate(read, schema)

    def test_backend_kinds_agree_with_schema(self):
        schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8"))
        active = next(option for option in schema["properties"]["active"]["oneOf"]
                      if option.get("type") == "object")
        kinds = active["properties"]["backend"]["properties"]["kind"]["enum"]
        assert set(kinds) == sb._BACKEND_KINDS

    @pytest.mark.parametrize("kind,endpoint", [
        ("llama-server", "llama-server-default"),
        ("hipfire", "llama-server-default"),
        ("unknown", "llama-server-default"),
        ("vllm", "vllm-default"),
    ])
    def test_default_endpoint_per_backend_kind(self, kind, endpoint):
        assert sb.default_endpoint_id(kind) == endpoint

    def test_retired_kind_has_no_default_endpoint(self):
        with pytest.raises(KeyError):
            sb.default_endpoint_id("lemonade")

    def test_state_file_is_readable_by_model_router_user(self, tmp_path):
        path = tmp_path / "model-state.json"
        _record(path)
        if os.name != "nt":
            assert stat.S_IMODE(path.stat().st_mode) == sb.STATE_FILE_MODE == 0o644

    def test_seq_monotonic_routeseq_only_on_change(self, tmp_path):
        path = tmp_path / "model-state.json"
        _record(path)
        doc = _record(path)  # identical route: seq moves, routeSeq does not
        assert doc["seq"] == 2 and doc["routeSeq"] == 1 and doc["history"] == []
        doc = _record(path, model="phi-4-mini", runtime="Phi-4-mini-Q4_K_M.gguf")
        assert doc["seq"] == 3 and doc["routeSeq"] == 2
        assert doc["history"][0]["runtimeModelId"] == "Qwen3.5-9B-Q4_K_M.gguf"
        assert doc["history"][0]["routeSeq"] == 1

    def test_history_bounded(self, tmp_path):
        path = tmp_path / "model-state.json"
        for index in range(13):
            _record(path, model=f"m-{index}", runtime=f"M-{index}.gguf")
        doc, errors = sb.read_state(path)
        assert errors == []
        assert len(doc["history"]) == sb.HISTORY_LIMIT
        assert doc["history"][0]["runtimeModelId"] == "M-11.gguf"

    def test_interrupted_write_keeps_original(self, tmp_path, monkeypatch):
        path = tmp_path / "model-state.json"
        _record(path)
        original = path.read_text(encoding="utf-8")
        real_replace = os.replace

        def exploding_replace(src, dst):
            raise OSError("simulated crash before rename")

        monkeypatch.setattr(sb.os, "replace", exploding_replace)
        with pytest.raises(OSError):
            _record(path, model="other", runtime="Other.gguf")
        monkeypatch.setattr(sb.os, "replace", real_replace)
        assert path.read_text(encoding="utf-8") == original
        assert not list(tmp_path.glob("*.tmp"))

    def test_concurrent_readers_never_see_partial(self, tmp_path):
        path = tmp_path / "model-state.json"
        _record(path)
        stop = threading.Event()
        problems: list[str] = []

        writer_error: list[str] = []

        def writer():
            try:
                for index in range(60):
                    _record(path, model=f"w-{index % 3}", runtime=f"W-{index % 3}.gguf")
            except Exception as exc:  # never hang pytest on a writer fault
                writer_error.append(repr(exc))
            finally:
                stop.set()

        thread = threading.Thread(target=writer)
        thread.start()
        while not stop.is_set():
            doc, errors = sb.read_state(path)
            if doc is None or sb.validate_state(doc):
                problems.append(f"bad read: {errors}")
                break
            time.sleep(0.001)
        thread.join(timeout=30)
        assert writer_error == []
        assert problems == []

    def test_malformed_state_is_diagnostic_never_promoted(self, tmp_path):
        path = tmp_path / "model-state.json"
        path.write_text("{not json", encoding="utf-8")
        doc, errors = sb.read_state(path)
        assert doc is None and errors
        with pytest.raises(sb.StateError):
            _record(path)
        assert sb.initialize_if_missing(path, {"GGUF_FILE": "X.gguf"}) is None
        assert path.read_text(encoding="utf-8") == "{not json"

    def test_validate_rejects_bad_shapes(self, tmp_path):
        doc = sb.initial_state()
        doc["seq"] = -1
        assert any("seq" in e for e in sb.validate_state(doc))
        doc = sb.initial_state()
        doc["operation"] = {"id": "x", "phase": "warp", "requestedModelId": "m", "startedAt": "t"}
        assert any("phase" in e for e in sb.validate_state(doc))
        doc = _record(tmp_path / "s.json")
        doc["active"]["backend"]["kind"] = "banana"
        assert any("backend.kind" in e for e in sb.validate_state(doc))
        doc = _record(tmp_path / "extra.json")
        doc["unexpected"] = True
        assert any("unexpected keys" in e for e in sb.validate_state(doc))
        doc = _record(tmp_path / "negative.json")
        doc["active"]["routeSeq"] = -1
        assert any("active.routeSeq" in e for e in sb.validate_state(doc))

    @pytest.mark.parametrize(
        "env,expected",
        [
            ({"GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf", "LLM_MODEL": "qwen3.5-9b"},
             {"catalogId": "qwen3.5-9b", "runtimeModelId": "Qwen3.5-9B-Q4_K_M.gguf", "backendKind": "llama-server"}),
            # A pre-round-F Lemonade .env: the GGUF filename is the runtime
            # identity (--alias), never the retired LEMONADE_MODEL id.
            ({"LEMONADE_MODEL": "extra.Qwen3.5-9B-Q4_K_M.gguf", "LLM_BACKEND": "lemonade",
              "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf"},
             {"catalogId": "Qwen3.5-9B-Q4_K_M", "runtimeModelId": "Qwen3.5-9B-Q4_K_M.gguf", "backendKind": "llama-server"}),
            ({"GGUF_FILE": "M.gguf", "AMD_INFERENCE_RUNTIME": "lemonade"},
             {"catalogId": "M", "runtimeModelId": "M.gguf", "backendKind": "llama-server"}),
            ({"LLM_MODEL": "native-model"},
             {"catalogId": "native-model", "runtimeModelId": "native-model", "backendKind": "llama-server"}),
            # A leftover LEMONADE_MODEL line never relabels the route.
            ({
                "GPU_BACKEND": "amd",
                "LLM_BACKEND": "llama-server",
                "LEMONADE_MODEL": "Qwen3.6-35B-A3B-GGUF",
                "LLM_MODEL": "qwen3.5-9b",
                "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
            }, {
                "catalogId": "qwen3.5-9b",
                "runtimeModelId": "Qwen3.5-9B-Q4_K_M.gguf",
                "backendKind": "llama-server",
            }),
            ({
                "LLM_BACKEND": "lemonade",
                "AMD_INFERENCE_MANAGED": "off",
                "LEMONADE_MODEL": "portable-model",
                "LLM_MODEL": "stale-model",
                "GGUF_FILE": "stale.gguf",
            }, {
                "catalogId": "stale-model",
                "runtimeModelId": "stale.gguf",
                "backendKind": "llama-server",
            }),
            # vLLM serves --served-model-name; a GGUF_FILE left by an earlier
            # llama-server install never names the route.
            ({
                "LLM_BACKEND": "vllm",
                "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4",
                "LLM_MODEL": "glm-5.3-flash-nvfp4",
                "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
                "MAX_CONTEXT": "131072",
            }, {
                "catalogId": "glm-5.3-flash-nvfp4",
                "runtimeModelId": "GLM-5.3-Flash-NVFP4",
                "backendKind": "vllm",
                "contextLength": 131072,
            }),
            ({"LLM_BACKEND": "vllm", "LLM_MODEL": "glm-5.3-flash-nvfp4",
              "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf"},
             {"catalogId": "glm-5.3-flash-nvfp4", "runtimeModelId": "glm-5.3-flash-nvfp4",
              "backendKind": "vllm"}),
            ({"LLM_BACKEND": " VLLM ", "VLLM_SERVED_MODEL": "Served-Model",
              "CTX_SIZE": "65536", "MAX_CONTEXT": "131072"},
             {"catalogId": "Served-Model", "runtimeModelId": "Served-Model",
              "backendKind": "vllm", "contextLength": 65536}),
            # VLLM_SERVED_MODEL names only a vllm route.
            ({"LLM_BACKEND": "llama-server", "VLLM_SERVED_MODEL": "Served-Model",
              "GGUF_FILE": "M.gguf"},
             {"catalogId": "M", "runtimeModelId": "M.gguf", "backendKind": "llama-server"}),
        ],
    )
    def test_migrate_env_forms(self, env, expected):
        got = sb.migrate_env_identity(env)
        for key, value in expected.items():
            assert got[key] == value

    def test_migrate_vllm_env_never_takes_the_gguf_identity(self):
        assert sb.migrate_env_identity({
            "LLM_BACKEND": "vllm", "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
        }) is None

    def test_migrate_cloud_only_yields_none(self):
        env = {
            "ODS_MODE": "cloud",
            "LLM_MODEL": "anthropic/claude-sonnet-4-6",
            "GGUF_FILE": "",
            "MAX_CONTEXT": "200000",
        }
        assert sb.migrate_env_identity(env) is None

    def test_initialize_if_missing_reconstructs_once(self, tmp_path):
        path = tmp_path / "model-state.json"
        env = {"GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf", "LLM_MODEL": "qwen3.5-9b", "MAX_CONTEXT": "131072"}
        doc = sb.initialize_if_missing(path, env)
        assert doc["active"]["reconstructed"] is True
        assert doc["active"]["verifiedAt"] is None
        assert doc["active"]["proof"]["completion"] is False
        assert doc["active"]["capabilities"]["agentViable"] is True
        assert doc["active"]["backend"]["endpointId"] == "llama-server-default"
        before = path.read_text(encoding="utf-8")
        assert sb.initialize_if_missing(path, {"GGUF_FILE": "Other.gguf"}) is None
        assert path.read_text(encoding="utf-8") == before

    def test_initialize_prefers_canonical_context_when_upgrade_aliases_diverge(self, tmp_path):
        path = tmp_path / "model-state.json"
        doc = sb.initialize_if_missing(
            path,
            {
                "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
                "LLM_MODEL": "qwen3.5-9b",
                "CTX_SIZE": "131072",
                "MAX_CONTEXT": "65536",
            },
        )

        assert doc["active"]["contextLength"] == 131072
        assert doc["active"]["capabilities"]["agentViable"] is True

    def test_initialize_skips_invalid_canonical_context_for_valid_legacy_alias(self, tmp_path):
        path = tmp_path / "model-state.json"
        doc = sb.initialize_if_missing(
            path,
            {
                "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
                "LLM_MODEL": "qwen3.5-9b",
                "CTX_SIZE": "auto",
                "MAX_CONTEXT": "65536",
            },
        )

        assert doc["active"]["contextLength"] == 65536
        assert doc["active"]["capabilities"]["agentViable"] is True

    def test_initialize_for_a_legacy_lemonade_env_uses_the_llama_server_endpoint(self, tmp_path):
        path = tmp_path / "model-state.json"
        doc = sb.initialize_if_missing(
            path,
            {
                "GPU_BACKEND": "amd",
                "LLM_BACKEND": "lemonade",
                "LEMONADE_MODEL": "extra.Model.gguf",
                "GGUF_FILE": "Model.gguf",
            },
        )
        assert doc["active"]["backend"] == {
            "kind": "llama-server", "endpointId": "llama-server-default", "nativeRoute": None,
        }
        assert doc["active"]["runtimeModelId"] == "Model.gguf"

    @pytest.mark.parametrize("env,kind,endpoint", [
        ({"GGUF_FILE": "M.gguf"}, "llama-server", "llama-server-default"),
        ({"LLM_BACKEND": "hipfire", "LLM_MODEL": "Qwen3.5-2B"}, "hipfire", "llama-server-default"),
        ({"LLM_BACKEND": "external", "LLM_MODEL": "lan-model"}, "unknown", "llama-server-default"),
        ({"LLM_BACKEND": "vllm", "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4"}, "vllm", "vllm-default"),
    ])
    def test_initialize_uses_the_backend_kinds_default_endpoint(self, tmp_path, env, kind, endpoint):
        doc = sb.initialize_if_missing(tmp_path / "model-state.json", env)
        assert doc["active"]["backend"] == {"kind": kind, "endpointId": endpoint, "nativeRoute": None}
        assert sb.validate_state(doc) == []

    def test_initialize_for_a_vllm_env_reconstructs_the_served_model(self, tmp_path):
        path = tmp_path / "model-state.json"
        doc = sb.initialize_if_missing(path, {
            "LLM_BACKEND": "vllm",
            "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4",
            "LLM_MODEL": "glm-5.3-flash-nvfp4",
            "GGUF_FILE": "Qwen3.5-9B-Q4_K_M.gguf",
            "MAX_CONTEXT": "131072",
        })
        assert doc["active"]["catalogId"] == "glm-5.3-flash-nvfp4"
        assert doc["active"]["runtimeModelId"] == "GLM-5.3-Flash-NVFP4"
        assert doc["active"]["contextLength"] == 131072
        assert doc["active"]["reconstructed"] is True
        assert doc["active"]["proof"] == {"identity": "GLM-5.3-Flash-NVFP4", "completion": False}
        read, errors = sb.read_state(path)
        assert errors == [] and read == doc
        jsonschema = pytest.importorskip("jsonschema")
        jsonschema.validate(read, json.loads(_SCHEMA_PATH.read_text(encoding="utf-8")))

    def test_initialize_keeps_a_caller_named_endpoint(self, tmp_path):
        doc = sb.initialize_if_missing(
            tmp_path / "model-state.json",
            {"LLM_BACKEND": "vllm", "VLLM_SERVED_MODEL": "GLM-5.3-Flash-NVFP4"},
            endpoint_id="vllm-native",
        )
        assert doc["active"]["backend"] == {
            "kind": "vllm", "endpointId": "vllm-native", "nativeRoute": None,
        }

    def test_legacy_lemonade_route_is_readable_but_never_written(self, tmp_path):
        path = tmp_path / "model-state.json"
        doc = _record(path, runtime="Model.gguf")
        doc["active"]["backend"] = {
            "kind": "lemonade", "endpointId": "lemonade-default", "nativeRoute": "extra.Model.gguf",
        }
        doc["active"]["runtimeModelId"] = "extra.Model.gguf"
        doc["active"]["proof"]["identity"] = "extra.Model.gguf"
        path.write_text(json.dumps(doc), encoding="utf-8")
        legacy, errors = sb.read_state(path)
        # Readable for one release so the host agent can migrate it.
        assert errors == [] and legacy["active"]["backend"]["kind"] == "lemonade"
        assert sb.is_legacy_route(legacy["active"]) is True
        assert sb.is_legacy_route(_record(tmp_path / "current.json")["active"]) is False
        before = path.read_text(encoding="utf-8")
        for backend, endpoint in (("lemonade", "llama-server-default"), ("llama-server", "lemonade-default")):
            with pytest.raises(sb.StateError, match="readable only"):
                _record(path, runtime="Model.gguf", backend=backend, endpoint=endpoint)
        assert path.read_text(encoding="utf-8") == before

    def test_initialize_cloud_only_writes_nothing(self, tmp_path):
        path = tmp_path / "model-state.json"
        env = {
            "ODS_MODE": "cloud",
            "LLM_MODEL": "anthropic/claude-sonnet-4-6",
            "MAX_CONTEXT": "200000",
        }
        assert sb.initialize_if_missing(path, env) is None
        assert not path.exists()

    def test_reconstructed_route_never_enters_verified_history(self, tmp_path):
        path = tmp_path / "model-state.json"
        reconstructed = sb.initialize_if_missing(
            path,
            {
                "ODS_MODE": "local",
                "LLM_MODEL": "stale-model",
                "GGUF_FILE": "Stale.gguf",
                "MAX_CONTEXT": "65536",
            },
        )
        assert reconstructed["active"]["proof"]["completion"] is False

        verified = _record(
            path,
            model="current-model",
            runtime="Current.gguf",
            context=65536,
        )
        assert verified["active"]["proof"]["completion"] is True
        assert verified["history"] == []


class TestModelStateEndpoint:
    def _point_at(self, monkeypatch, tmp_path):
        data_dir = tmp_path / "mounted-data"
        data_dir.mkdir(parents=True, exist_ok=True)
        monkeypatch.setenv("ODS_DATA_DIR", str(data_dir))
        monkeypatch.setenv("ODS_MODEL_STATE_SCHEMA_PATH", str(_SCHEMA_PATH))
        return data_dir / "model-state.json"

    def test_state_path_uses_data_mount_environment(self, monkeypatch, tmp_path):
        from routers import model_state as ms

        data_dir = tmp_path / "compose-data-mount"
        monkeypatch.setenv("ODS_DATA_DIR", str(data_dir))
        monkeypatch.setattr(ms, "INSTALL_DIR", str(tmp_path / "ods"))
        assert ms._state_path() == data_dir / "model-state.json"

    def test_missing_state(self, test_client, monkeypatch, tmp_path):
        self._point_at(monkeypatch, tmp_path)
        resp = test_client.get("/api/models/state", headers=test_client.auth_headers)
        assert resp.status_code == 200
        body = resp.json()
        assert body["exists"] is False and body["valid"] is False
        assert body["active"] is None and body["historyCount"] == 0

    def test_valid_state_summary(self, test_client, monkeypatch, tmp_path):
        path = self._point_at(monkeypatch, tmp_path)
        _record(path, context=131072)
        _record(path, model="phi-4-mini", runtime="Phi-4-mini-Q4_K_M.gguf", context=131072)
        resp = test_client.get("/api/models/state", headers=test_client.auth_headers)
        assert resp.status_code == 200
        body = resp.json()
        assert body["exists"] is True and body["valid"] is True
        assert body["routeSeq"] == 2
        assert body["active"]["catalogId"] == "phi-4-mini"
        assert body["historyCount"] == 1
        assert body["capabilityImpact"]["agentViable"] is True

    def test_malformed_state_is_diagnostic(self, test_client, monkeypatch, tmp_path):
        path = self._point_at(monkeypatch, tmp_path)
        path.write_text("][", encoding="utf-8")
        resp = test_client.get("/api/models/state", headers=test_client.auth_headers)
        assert resp.status_code == 200
        body = resp.json()
        assert body["exists"] is True and body["valid"] is False and body["errors"]

    def test_schema_invalid_state_is_diagnostic(self, test_client, monkeypatch, tmp_path):
        path = self._point_at(monkeypatch, tmp_path)
        doc = _record(path)
        doc["active"]["routeSeq"] = -1
        doc["unexpected"] = "must be rejected"
        path.write_text(json.dumps(doc), encoding="utf-8")

        resp = test_client.get("/api/models/state", headers=test_client.auth_headers)
        assert resp.status_code == 200
        body = resp.json()
        assert body["exists"] is True
        assert body["valid"] is False
        assert body["active"] is None
        assert any("routeSeq" in error for error in body["errors"])
        assert any("unexpected" in error for error in body["errors"])

    def test_non_dict_root_state_is_diagnostic(self, test_client, monkeypatch, tmp_path):
        path = self._point_at(monkeypatch, tmp_path)
        path.write_text('["array", "root"]', encoding="utf-8")
        resp = test_client.get("/api/models/state", headers=test_client.auth_headers)
        assert resp.status_code == 200
        body = resp.json()
        assert body["exists"] is True and body["valid"] is False
        assert any("JSON object" in err for err in body["errors"])

    def test_non_utf8_binary_state_is_diagnostic(self, test_client, monkeypatch, tmp_path):
        path = self._point_at(monkeypatch, tmp_path)
        path.write_bytes(b"\x80\xff\xfe\xfd")
        resp = test_client.get("/api/models/state", headers=test_client.auth_headers)
        assert resp.status_code == 200
        body = resp.json()
        assert body["exists"] is True and body["valid"] is False
        assert any("read failed" in err for err in body["errors"])

    def test_requires_auth(self, test_client, monkeypatch, tmp_path):
        self._point_at(monkeypatch, tmp_path)
        resp = test_client.get("/api/models/state")
        assert resp.status_code in (401, 403)


class TestObserveHook:
    @pytest.fixture(autouse=True)
    def _isolation(self, monkeypatch, tmp_path):
        import test_model_activate as tma
        config_dir = tmp_path / "isolated-home" / ".config" / "opencode"
        monkeypatch.setattr(
            tma._mod,
            "_opencode_config_paths",
            lambda: (config_dir / "opencode.json", config_dir / "config.json"),
        )
        monkeypatch.setattr(tma._mod, "_chat_completion_ready", lambda *_a, **_k: True)
        monkeypatch.setattr(tma._mod, "_runtime_health", lambda _env: "ok")
        monkeypatch.setattr(tma._mod, "_llama_runtime_props", lambda _env: (65536, "", None))
        monkeypatch.setattr(tma._mod, "_container_exists", lambda _c: False)
        monkeypatch.setattr(tma._mod, "_container_running", lambda _c: False)
        monkeypatch.setattr(
            tma._mod,
            "_capture_container_state",
            lambda container: {"exists": False, "running": False},
        )
        monkeypatch.setattr(tma._mod, "_wait_for_container_health", lambda _c: None)
        monkeypatch.setattr(
            tma._mod,
            "_capture_managed_opencode_state",
            lambda: {"system": tma._mod.platform.system(), "active": False},
        )
        monkeypatch.setattr(tma._mod, "_opencode_installed", lambda: False)

    def test_activation_success_records_state(self, tmp_path, monkeypatch):
        import subprocess
        import test_model_activate as tma

        install_dir = tma._write_model_activation_fixture(tmp_path)[0]
        monkeypatch.setattr(tma._mod, "INSTALL_DIR", install_dir)
        monkeypatch.delenv("ODS_HOST_INSTALL_DIR", raising=False)
        monkeypatch.setattr(tma._mod.time, "sleep", lambda _s: None)
        monkeypatch.setattr(tma._mod, "_compose_restart_llama_server", lambda _env: None)

        def fake_run(cmd, **_kwargs):
            stdout = tma._llama_identity_response("new-model.gguf") if cmd and cmd[0] == "curl" else ""
            return subprocess.CompletedProcess(cmd, 0, stdout=stdout, stderr="")

        monkeypatch.setattr(tma._mod.subprocess, "run", fake_run)
        assert tma._mod._switchboard_state is not None

        handler = tma._ResponseHandler()
        tma._mod.AgentHandler._do_model_activate(handler, "target-model")
        assert handler.response_code == 200

        state_path = install_dir / "data" / "model-state.json"
        doc, errors = sb.read_state(state_path)
        assert errors == [] and doc is not None
        assert doc["routeSeq"] == 1
        assert doc["active"]["catalogId"] == "target-model"
        assert doc["active"]["runtimeModelId"] == "new-model.gguf"
        assert doc["active"]["proof"]["completion"] is True

    def test_activation_failure_leaves_state_untouched(self, tmp_path, monkeypatch):
        import subprocess
        import test_model_activate as tma

        install_dir = tma._write_model_activation_fixture(tmp_path)[0]
        state_path = install_dir / "data" / "model-state.json"
        _record(state_path, model="previous", runtime="Previous.gguf")
        before = state_path.read_text(encoding="utf-8")

        monkeypatch.setattr(tma._mod, "INSTALL_DIR", install_dir)
        monkeypatch.delenv("ODS_HOST_INSTALL_DIR", raising=False)
        monkeypatch.setattr(tma._mod.time, "sleep", lambda _s: None)
        monkeypatch.setattr(tma._mod, "_compose_restart_llama_server", lambda _env: None)

        def failing_run(cmd, **_kwargs):
            # Identity probe returns the wrong model so verification fails.
            stdout = tma._llama_identity_response("wrong-model.gguf") if cmd and cmd[0] == "curl" else ""
            return subprocess.CompletedProcess(cmd, 0, stdout=stdout, stderr="")

        monkeypatch.setattr(tma._mod.subprocess, "run", failing_run)
        handler = tma._ResponseHandler()
        tma._mod.AgentHandler._do_model_activate(handler, "target-model")
        assert handler.response_code != 200
        assert state_path.read_text(encoding="utf-8") == before

    def test_initial_reconstructed_state_promotes_after_readiness_proof(
        self, tmp_path, monkeypatch
    ):
        import test_model_activate as tma

        install_dir = tma._write_model_activation_fixture(tmp_path)[0]
        state_path = install_dir / "data" / "model-state.json"
        monkeypatch.setattr(tma._mod, "INSTALL_DIR", install_dir)

        env = tma._mod.load_env(install_dir / ".env")
        reconstructed = sb.initialize_if_missing(state_path, env)
        assert reconstructed["active"]["reconstructed"] is True
        assert reconstructed["active"]["proof"]["completion"] is False

        monkeypatch.setattr(
            tma._mod,
            "_wait_for_model_readiness",
            lambda *_args, **_kwargs: {
                "identity": "old-model.gguf",
                "contextLength": 65536,
                "contextVerified": True,
                "verifiedAt": "2026-07-20T00:00:00Z",
            },
        )

        assert tma._mod._publish_verified_initial_switchboard_route(
            reason="test", attempts=1, initial_delay=0, interval=0
        ) is True
        doc, errors = sb.read_state(state_path)
        assert errors == [] and doc is not None
        assert doc["active"].get("reconstructed") is None
        assert doc["active"]["runtimeModelId"] == "old-model.gguf"
        assert doc["active"]["verifiedAt"]
        assert doc["active"]["proof"] == {
            "identity": "old-model.gguf",
            "completion": True,
        }

    def test_legacy_lemonade_env_initial_route_proves_the_gguf_alias(
        self, tmp_path, monkeypatch
    ):
        import test_model_activate as tma

        install_dir = tma._write_model_activation_fixture(tmp_path)[0]
        env_path = install_dir / ".env"
        env_path.write_text(
            "\n".join([
                "ODS_MODE=lemonade",
                "GPU_BACKEND=amd",
                "LLM_BACKEND=lemonade",
                "AMD_INFERENCE_RUNTIME=lemonade",
                "LEMONADE_MODEL=extra.Qwen3.5-9B-Q4_K_M.gguf",
                "LLM_MODEL=qwen3.5-9b",
                "GGUF_FILE=Qwen3.5-9B-Q4_K_M.gguf",
                "CTX_SIZE=65536",
                "",
            ]),
            encoding="utf-8",
        )
        state_path = install_dir / "data" / "model-state.json"
        reconstructed = sb.initialize_if_missing(state_path, tma._mod.load_env(env_path))
        assert reconstructed["active"]["backend"]["kind"] == "llama-server"

        readiness_calls = []

        def readiness(*_args, **kwargs):
            readiness_calls.append(kwargs)
            return {
                "identity": "Qwen3.5-9B-Q4_K_M.gguf",
                "contextLength": 65536,
                "contextVerified": True,
                "verifiedAt": "2026-09-04T00:00:00Z",
            }

        monkeypatch.setattr(tma._mod, "INSTALL_DIR", install_dir)
        monkeypatch.setattr(tma._mod, "_wait_for_model_readiness", readiness)

        assert tma._mod._publish_verified_initial_switchboard_route(
            reason="legacy-env-test", attempts=1, initial_delay=0, interval=0
        ) is True
        assert readiness_calls[0]["gguf_file"] == "Qwen3.5-9B-Q4_K_M.gguf"
        assert "lemonade_model_id" not in readiness_calls[0]

        doc, errors = sb.read_state(state_path)
        assert errors == [] and doc is not None
        assert doc["active"]["runtimeModelId"] == "Qwen3.5-9B-Q4_K_M.gguf"
        assert doc["active"]["backend"] == {
            "kind": "llama-server",
            "endpointId": "llama-server-default",
            "nativeRoute": None,
        }

    @pytest.mark.parametrize(
        "key,new_value",
        [
            ("GPU_BACKEND", "nvidia"),
            ("GGUF_FILE", "Other.gguf"),
        ],
    )
    def test_initial_route_discards_proof_after_runtime_contract_changes(
        self, tmp_path, monkeypatch, key, new_value
    ):
        import test_model_activate as tma

        install_dir = tma._write_model_activation_fixture(tmp_path)[0]
        env_path = install_dir / ".env"
        original_values = {
            "ODS_MODE": "local",
            "GPU_BACKEND": "amd",
            "LLM_BACKEND": "llama-server",
            "AMD_INFERENCE_RUNTIME": "llama-server",
            "GGUF_FILE": "portable-model.gguf",
            "CTX_SIZE": "65536",
        }
        original = "".join(f"{name}={value}\n" for name, value in original_values.items())
        env_path.write_text(original, encoding="utf-8")
        state_path = install_dir / "data" / "model-state.json"
        sb.initialize_if_missing(state_path, tma._mod.load_env(env_path))

        def readiness(*_args, **_kwargs):
            changed_values = dict(original_values)
            changed_values[key] = new_value
            env_path.write_text(
                "".join(f"{name}={value}\n" for name, value in changed_values.items()),
                encoding="utf-8",
            )
            return {
                "identity": "portable-model.gguf",
                "contextLength": 65536,
                "contextVerified": True,
                "verifiedAt": "2026-09-04T00:00:00Z",
            }

        monkeypatch.setattr(tma._mod, "INSTALL_DIR", install_dir)
        monkeypatch.setattr(tma._mod, "_wait_for_model_readiness", readiness)

        assert tma._mod._publish_verified_initial_switchboard_route(
            reason="runtime-race-test", attempts=1, initial_delay=0, interval=0
        ) is False
        doc, errors = sb.read_state(state_path)
        assert errors == [] and doc is not None
        assert doc["active"]["reconstructed"] is True

    def test_initial_reconstructed_state_stays_unroutable_without_proof(
        self, tmp_path, monkeypatch
    ):
        import test_model_activate as tma

        install_dir = tma._write_model_activation_fixture(tmp_path)[0]
        state_path = install_dir / "data" / "model-state.json"
        monkeypatch.setattr(tma._mod, "INSTALL_DIR", install_dir)

        env = tma._mod.load_env(install_dir / ".env")
        sb.initialize_if_missing(state_path, env)
        before = state_path.read_text(encoding="utf-8")
        monkeypatch.setattr(
            tma._mod,
            "_wait_for_model_readiness",
            lambda *_args, **_kwargs: {},
        )

        assert tma._mod._publish_verified_initial_switchboard_route(
            reason="test", attempts=1, initial_delay=0, interval=0
        ) is False
        assert state_path.read_text(encoding="utf-8") == before

    def test_initial_route_proof_defers_during_model_activation(
        self, tmp_path, monkeypatch
    ):
        import test_model_activate as tma

        install_dir = tma._write_model_activation_fixture(tmp_path)[0]
        state_path = install_dir / "data" / "model-state.json"
        monkeypatch.setattr(tma._mod, "INSTALL_DIR", install_dir)
        sb.initialize_if_missing(state_path, tma._mod.load_env(install_dir / ".env"))
        monkeypatch.setattr(
            tma._mod,
            "_wait_for_model_readiness",
            lambda *_args, **_kwargs: (_ for _ in ()).throw(
                AssertionError("active lifecycle reached initial route readiness")
            ),
        )
        monkeypatch.setattr(
            tma._mod,
            "_switchboard_initial_verify_cancel",
            tma._mod.threading.Event(),
        )
        monkeypatch.setattr(tma._mod, "_model_lifecycle_operation", "model_activation")
        monkeypatch.setattr(tma._mod, "_model_lifecycle_target", "target-model")

        assert tma._mod._publish_verified_initial_switchboard_route(
            reason="test", attempts=1, initial_delay=0, interval=0
        ) is False

    def test_cancelled_initial_route_readiness_never_probes_the_runtime(
        self, tmp_path, monkeypatch
    ):
        import test_model_activate as tma

        install_dir = tma._write_model_activation_fixture(tmp_path)[0]
        monkeypatch.setattr(tma._mod, "INSTALL_DIR", install_dir)
        monkeypatch.setattr(
            tma._mod.subprocess,
            "run",
            lambda *_args, **_kwargs: (_ for _ in ()).throw(
                AssertionError("cancelled readiness reached a runtime probe")
            ),
        )
        cancel_event = tma._mod.threading.Event()
        cancel_event.set()

        assert tma._mod._wait_for_model_readiness(
            {
                "GPU_BACKEND": "amd",
                "AMD_INFERENCE_LOCATION": "host",
                "AMD_INFERENCE_PORT": "8080",
            },
            model_id="qwen3-4b-instruct-2507-q4",
            gguf_file="Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
            llm_model_name="Qwen3-4B-Instruct-2507-Q4_K_M",
            attempts=60,
            initial_delay=0,
            interval=5,
            return_proof=True,
            cancel_event=cancel_event,
        ) == {}