ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
"""Image policy must have evidence for the exact activated model route."""
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import pytest
_path = Path(__file__).resolve().parents[4] / "bin" / "ods-host-agent.py"
_spec = importlib.util.spec_from_file_location("ods_host_image_input", _path)
agent = importlib.util.module_from_spec(_spec)
sys.modules[_spec.name] = agent
_spec.loader.exec_module(agent)
@pytest.mark.parametrize("record,expected", [
({"id": "selected", "vision": True}, "supported"),
({"id": "selected", "vision": False}, "unsupported"),
({"id": "selected"}, "unknown"),
({"id": "selected", "vision": "false"}, "unknown"),
({"id": "selected", "vision": 0}, "unknown"),
({"id": "other", "vision": True}, "unknown"),
({"id": "canonical", "llm_model_name": "selected", "vision": True}, "supported"),
])
def test_only_explicit_exact_curated_capability_is_evidence(tmp_path, monkeypatch, record, expected):
monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
(tmp_path / "config").mkdir()
(tmp_path / "config/model-library.json").write_text(json.dumps({"models": [record]}))
monkeypatch.setattr(agent, "_load_model_library_records", lambda: pytest.fail("normalized/imported defaults are not evidence"))
assert agent._pixel_model_image_input("selected") == expected
def test_ambiguous_missing_and_invalid_catalog_are_unknown(tmp_path, monkeypatch):
monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
assert agent._pixel_model_image_input("selected") == "unknown"
(tmp_path / "config").mkdir()
catalog = tmp_path / "config/model-library.json"
catalog.write_text(json.dumps({"models": [
{"id": "selected", "vision": True}, {"gguf_file": "selected", "vision": False},
]}))
assert agent._pixel_model_image_input("selected") == "unknown"
catalog.write_text("{")
assert agent._pixel_model_image_input("selected") == "unknown"
def test_shell_transport_changes_and_clears_policy_without_reusing_previous_model(tmp_path, monkeypatch):
monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
monkeypatch.setattr(agent, "_ods_managed_pixel_identity", lambda: ("owner", tmp_path / "home"))
monkeypatch.setattr(agent, "load_env", lambda _: {})
calls = []
def run(command, **_kwargs):
calls.append(command)
return subprocess.CompletedProcess(command, 0, "", "")
monkeypatch.setattr(agent.subprocess, "run", run)
common = {"contextLength": 32768, "maxTokens": 4096, "reasoning": False}
for model, policy in [("vision", "supported"), ("text", "unsupported"), ("unidentified", None)]:
contract = {**common, "model": model}
if policy is not None:
contract["imageInput"] = policy
assert agent._reconcile_managed_pixel_contract(contract) == "reconciled"
assert calls[-1][-2:] == [policy or "unknown", ""]
assert '"$target_route_fingerprint" "" "$target_image_input"' in calls[-1][2]
with pytest.raises(RuntimeError, match="image-input policy"):
agent._reconcile_managed_pixel_contract({**common, "model": "vision", "imageInput": True})
assert len(calls) == 3
def test_runtime_readback_retains_policy_and_rejects_boolean_default(tmp_path, monkeypatch):
monkeypatch.setattr(agent, "INSTALL_DIR", tmp_path)
monkeypatch.setattr(agent, "_ods_managed_pixel_identity", lambda: ("owner", tmp_path / "home"))
snapshot = agent._snapshot_text_file
monkeypatch.setattr(agent, "_snapshot_text_file", lambda path: {**snapshot(path), "mode": 0o600})
onboarding = tmp_path / "data/pixel/onboarding.json"
onboarding.parent.mkdir(parents=True)
value = {"modelProvider": "ods-gateway", "modelId": "ods/current", "modelName": "ODS Current (vision)",
"modelContextWindow": 32768, "modelMaxTokens": 4096, "modelReasoning": False,
"modelImageInput": "supported"}
onboarding.write_text(json.dumps(value))
onboarding.chmod(0o600)
assert agent._managed_pixel_runtime_contract()["imageInput"] == "supported"
value["modelImageInput"] = False
onboarding.write_text(json.dumps(value))
with pytest.raises(RuntimeError, match="runtime contract"):
agent._managed_pixel_runtime_contract()
def test_legacy_remote_proof_checks_route_without_claiming_visual_verification(monkeypatch):
route = {"provider": {"model": "remote", "transport": "direct", "baseUrl": "https://example.invalid/v1",
"contextLength": 32768, "maxTokens": 4096, "reasoning": False}}
monkeypatch.setattr(agent, "_read_remote_provider_route_state_for_update", lambda: route)
verified = []
monkeypatch.setattr(agent, "_verify_litellm_route", lambda *_args, **kwargs: verified.append(kwargs))
contract = agent._remote_provider_runtime_contract(route)
assert contract["imageInput"] == "unknown"
legacy = {key: value for key, value in contract.items() if key != "imageInput"}
assert agent._prove_pixel_model_contract({}, legacy)
assert agent._prove_pixel_model_contract({}, {**contract, "imageInput": "supported"})
assert not agent._prove_pixel_model_contract({}, {**contract, "model": "another"})
assert not agent._prove_pixel_model_contract({}, {**contract, "routeFingerprint": "f" * 64})
assert not agent._prove_pixel_model_contract({}, {**contract, "imageInput": False})
assert verified == [{"model": "ods/current"}, {"model": "ods/current"}]
# What ODS's own llama-server reports in /props changes Pixel's declaration
# only when it says a vision projector is loaded (llama.cpp with an mmproj).
def _props_answer(monkeypatch, answer):
def runtime_http(_env, path, **_kwargs):
assert path == "/props"
if isinstance(answer, BaseException):
raise answer
return answer if isinstance(answer, str) else json.dumps(answer)
monkeypatch.setattr(agent, "_runtime_http", runtime_http)
_SERVED = {"default_generation_settings": {"n_ctx": 65536}, "model_path": "/models/Model.gguf"}
@pytest.mark.parametrize("modalities,expected", [
({"vision": True, "audio": False}, True),
({"vision": False, "audio": False}, False),
(None, None), # a build that reports no modalities
({"vision": "true"}, None),
({"vision": 1}, None),
({"audio": True}, None),
(True, None),
])
def test_props_report_vision_only_as_an_explicit_boolean(monkeypatch, modalities, expected):
_props_answer(monkeypatch, _SERVED if modalities is None else {**_SERVED, "modalities": modalities})
assert agent._llama_runtime_props({}) == (65536, "Model.gguf", expected)
@pytest.mark.parametrize("answer", [
OSError("llama-server /props is unreachable (curl exit 7)"),
subprocess.TimeoutExpired("curl", 5),
"{", "[]", "",
])
def test_an_unreadable_props_answer_reports_no_vision(monkeypatch, answer):
_props_answer(monkeypatch, answer)
assert agent._llama_runtime_props({}) == (0, "", None)
def _llama_answers(modalities):
props = dict(_SERVED) if modalities is None else {**_SERVED, "modalities": modalities}
return {
"/health": {"status": "ok"},
"/v1/models": {"object": "list", "data": [{"id": "Model.gguf", "object": "model"}]},
"/props": props,
"/v1/chat/completions": {"model": "Model.gguf", "choices": [{"message": {"content": "READY"}}]},
}
_KEY = "cd" * 32
# Every route whose model server is ODS's own llama-server, as its proof reaches it.
_ROUTES = {
"linux-host": ({"GPU_BACKEND": "nvidia", "OLLAMA_PORT": "8080"}, "http://127.0.0.1:8080"),
"container-agent": ({"GPU_BACKEND": "nvidia", "OLLAMA_PORT": "8080"}, "http://ods-llama-server:8080"),
"windows-native": ({"GPU_BACKEND": "amd", "AMD_INFERENCE_LOCATION": "host", "AMD_INFERENCE_RUNTIME": "llama-server",
"AMD_INFERENCE_PORT": "18080", "LLAMA_SERVER_API_KEY": _KEY}, "http://127.0.0.1:18080"),
"macos-native": ({"GPU_BACKEND": "apple", "ODS_NATIVE_LLAMA_PORT": "8080"}, "http://127.0.0.1:8080"),
"wsl-router": ({"GPU_BACKEND": "cpu", "ODS_HOST_LLM_TRANSPORT": "model-router", "LLAMA_SERVER_API_KEY": _KEY},
"http://host.docker.internal:8080"),
}
def _serve_route(monkeypatch, route, answers):
env, origin = _ROUTES[route]
requests: list = []
monkeypatch.delenv("ODS_HOST_INSTALL_DIR", raising=False)
if route == "container-agent":
monkeypatch.setenv("ODS_HOST_INSTALL_DIR", "/ods")
monkeypatch.setattr(agent.platform, "system", lambda: "Windows" if route == "windows-native" else "Linux")
monkeypatch.setattr(agent._wsl_runtime, "candidate", lambda _env: route == "wsl-router")
if route == "wsl-router":
def transport(_install, at, path, payload=None, api_key="", timeout=5):
assert api_key == _KEY
requests.append(at + path)
return json.dumps(answers[path])
monkeypatch.setattr(agent, "_router_transport_request", transport)
monkeypatch.setattr(agent.subprocess, "run", lambda *_a, **_k: pytest.fail("WSL reaches Windows only through the router"))
else:
def run(command, **_kwargs):
url = next(str(part) for part in command if str(part).startswith("http"))
requests.append(url)
return subprocess.CompletedProcess(command, 0, json.dumps(answers[url[len(origin):]]), "")
monkeypatch.setattr(agent.subprocess, "run", run)
return {**env, "CTX_SIZE": "65536"}, origin, requests
@pytest.mark.parametrize("route", sorted(_ROUTES))
@pytest.mark.parametrize("modalities,vision", [
({"vision": True, "audio": False}, True),
({"vision": False, "audio": False}, False),
(None, None),
])
def test_the_readiness_proof_carries_vision_from_its_own_props_read(monkeypatch, route, modalities, vision):
env, origin, requests = _serve_route(monkeypatch, route, _llama_answers(modalities))
proof = agent._wait_for_model_readiness(
env, model_id="model", gguf_file="Model.gguf", llm_model_name="model",
attempts=1, initial_delay=0, interval=0, return_proof=True, require_exact_context=True)
assert proof["identity"] == "Model.gguf" and proof["contextVerified"] is True
assert proof.get("vision", "absent") == ("absent" if vision is None else vision)
# No extra request: the vision report is the /props answer that proved the context.
assert [url[len(origin):] for url in requests] == ["/health", "/v1/models", "/props", "/v1/chat/completions"]
assert all(url.startswith(origin) for url in requests)
@pytest.mark.parametrize("proof,configured,expected", [
({"identity": "m", "vision": True}, "unknown", "supported"),
({"identity": "m", "vision": True}, "unsupported", "supported"),
({"identity": "m", "vision": False}, "unknown", "unknown"),
({"identity": "m", "vision": False}, "unsupported", "unsupported"),
({"identity": "m", "vision": False}, "supported", "supported"),
({"identity": "m"}, "unknown", "unknown"),
({"identity": "m", "vision": "true"}, "unknown", "unknown"),
({"identity": "m", "vision": 1}, "unknown", "unknown"),
({}, "unknown", "unknown"),
(None, "unsupported", "unsupported"),
])
def test_only_a_reported_vision_projector_declares_support(proof, configured, expected):
assert agent._pixel_image_input_readback(configured, proof) == expected
def test_a_remote_route_never_reads_the_local_vision_report(tmp_path, monkeypatch):
# Whatever the local llama-server would report, the remote provider's
# declaration stays unknown and the previous local one is kept for restore.
install, data = tmp_path / "install", tmp_path / "data"
install.mkdir()
(install / ".env").write_text("ODS_MODE=local\nLLM_API_URL=http://llama-server:8080\nLITELLM_KEY=k\n"
"PIXEL_OPENWEBUI_KEY=configured\n", encoding="utf-8")
monkeypatch.setattr(agent, "INSTALL_DIR", install)
monkeypatch.setattr(agent, "DATA_DIR", data)
local = {"model": "vision.gguf", "contextLength": 65536, "maxTokens": 8192, "reasoning": False,
"imageInput": "supported"}
events: list = []
class Transaction:
previous = dict(local)
completed = False
def apply(self, target):
events.append(("apply", dict(target)))
return "reconciled"
def finish(self, outcome):
events.append(("finish", outcome))
self.completed = True
monkeypatch.setattr(agent, "_managed_pixel_runtime_contract", lambda: dict(local))
monkeypatch.setattr(agent, "_begin_pixel_model_transaction", lambda _env: Transaction())
monkeypatch.setattr(agent, "_capture_container_state", lambda _name: {"exists": True, "running": True})
monkeypatch.setattr(agent, "_restart_existing_container", lambda *_a, **_k: True)
monkeypatch.setattr(agent, "_wait_for_container_health", lambda _name: None)
monkeypatch.setattr(agent, "_verify_litellm_route", lambda *_a, **_k: None)
monkeypatch.setattr(agent, "_render_remote_provider_cloud_config", lambda *_a: None)
for name in ("_runtime_http", "_llama_runtime_props", "_wait_for_model_readiness"):
monkeypatch.setattr(agent, name, lambda *_a, _name=name, **_k: pytest.fail(f"remote activation called {_name}"))
route = {"provider": {"model": "remote-vision", "transport": "direct", "baseUrl": "https://example.invalid/v1",
"contextLength": 32768, "maxTokens": 4096, "reasoning": False}}
activation = agent._activate_remote_provider_route(route)
assert activation["pixel"] == "reconciled"
assert events == [("apply", agent._remote_provider_runtime_contract(route)), ("finish", "commit")]
assert events[0][1]["imageInput"] == "unknown"
assert agent._read_remote_provider_activation_state()["previous"]["pixel"] == local