Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

39.4 KB
Raw Blame
#!/usr/bin/env python3
"""Tests for scripts/render-runtime-configs.py."""

from __future__ import annotations

import importlib.util
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from types import ModuleType


ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "scripts" / "render-runtime-configs.py"


def load_renderer_module() -> ModuleType:
    name = "ods_render_runtime_configs_test"
    spec = importlib.util.spec_from_file_location(name, SCRIPT)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    sys.modules[name] = module
    spec.loader.exec_module(module)
    return module


def run_renderer(*args: str, env: dict[str, str] | None = None) -> dict[str, object]:
    proc = subprocess.run(
        [sys.executable, str(SCRIPT), *args],
        cwd=ROOT,
        env=env,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        check=True,
    )
    return json.loads(proc.stdout)


def llm_backend_env(value: str | None = None) -> dict[str, str]:
    """This environment with LLM_BACKEND unset, or set to ``value``."""
    env = {key: item for key, item in os.environ.items() if key != "LLM_BACKEND"}
    if value is not None:
        env["LLM_BACKEND"] = value
    return env


# Byte for byte what the renderer wrote before LLM_BACKEND became an input.
LLAMA_SERVER_ENDPOINTS = (
    '{\n  "endpoints": [\n    {\n      "id": "llama-server-default",\n'
    '      "baseUrl": "http://llama-server:8080"\n    }\n  ]\n}\n'
)
LLAMA_SERVER_ENV = (
    "ODS_MODE=local\nODS_MODEL_SWITCHBOARD=enabled\nLLM_BACKEND=llama-server\n"
    "LLM_MODEL=qwen3.5-9b\nGGUF_FILE=Qwen3.5-9B-Q4_K_M.gguf\nGPU_BACKEND=nvidia\n"
    "LLM_API_URL=http://llama-server:8080/v1\nCTX_SIZE=131072\nMAX_CONTEXT=131072\n"
    "OPEN_WEBUI_LLM_BASE_URL=http://litellm:4000\nOPEN_WEBUI_LLM_API_KEY=sk-test-litellm\n"
)
ALL_ENABLED = ("--surface", "all", "--switchboard-mode", "enabled", "--litellm-key", "sk-test-litellm")


def file_by_surface(payload: dict[str, object], surface: str) -> dict[str, str]:
    for item in payload["files"]:
        if item["surface"] == surface:
            return item
    raise AssertionError(f"missing surface {surface}")


def model_provider_by_id(settings: dict[str, object], provider_id: str) -> dict[str, object]:
    for provider in settings["modelProviders"]:
        if provider["id"] == provider_id:
            return provider
    raise AssertionError(f"missing model provider {provider_id}")


def test_external_model_uses_authenticated_gateway_without_vendor_impersonation() -> None:
    result = run_renderer("--surface", "litellm-external", "--model", "owner/model-q4",
                          "--llm-base-url", "http://10.0.2.2:18080")
    rendered = file_by_surface(result, "litellm-external")
    assert rendered["path"] == "config/litellm/local.yaml"
    config = rendered["content"]
    assert 'model_name: "ods/current"' in config
    assert config.count('model: "openai/owner/model-q4"') == 4
    assert config.count('api_base: "http://10.0.2.2:18080/v1"') == 4
    assert "master_key: os.environ/LITELLM_MASTER_KEY" in config
    assert "enable_thinking" not in config  # Do not invent backend-specific capabilities.
    assert config.count("api_key: not-needed") == 4


def test_external_gateway_rejects_credentialed_or_malformed_bases() -> None:
    for base in ("file:///tmp/model", "http://user:secret@host/v1", "http://host?key=x",
                 "http://host:99999", "http://host/v1#fragment", "http://host\n/v1"):
        result = subprocess.run([sys.executable, str(SCRIPT), "--surface", "litellm-external",
                                 "--llm-base-url", base], capture_output=True, text=True)
        assert result.returncode != 0, base
    result = run_renderer("--surface", "litellm-external", "--model", 'owner/"model',
                          "--llm-base-url", "http://[::1]:18080/v1")
    content = file_by_surface(result, "litellm-external")["content"]
    assert 'model: ' + json.dumps('openai/owner/"model') in content


def test_external_gateway_uses_runtime_key_reference_when_authenticated() -> None:
    result = run_renderer(
        "--surface", "litellm-external", "--model", "test-model",
        "--llm-base-url", "https://upstream.example/v1",
        "--external-llm-authenticated",
    )
    config = file_by_surface(result, "litellm-external")["content"]
    assert config.count("api_key: os.environ/EXTERNAL_LLM_API_KEY") == 4
    assert "api_key: not-needed" not in config
    assert "test-secret-123" not in config


def test_all_surfaces_render() -> None:
    payload = run_renderer("--surface", "all")
    surfaces = {item["surface"] for item in payload["files"]}
    assert surfaces == {
        "env", "opencode", "litellm-local", "perplexica", "hermes",
        "model-router-endpoints", "litellm-switchboard",
    }
    assert payload["mode"] == "dry-run"


def test_fresh_default_uses_stable_switchboard_alias() -> None:
    payload = run_renderer("--surface", "all")
    assert payload["inputs"]["switchboard_mode"] == "enabled"
    switchboard = file_by_surface(payload, "litellm-switchboard")["content"]
    assert "api_base: http://model-router:9099/v1" in switchboard


def test_switchboard_surface_gated_on_enabled_mode() -> None:
    observed = run_renderer("--surface", "all", "--switchboard-mode", "observe")
    assert "litellm-switchboard" not in {i["surface"] for i in observed["files"]}
    enabled = run_renderer("--surface", "all", "--switchboard-mode", "enabled")
    surfaces = {i["surface"] for i in enabled["files"]}
    assert "litellm-switchboard" in surfaces
    switchboard = next(i for i in enabled["files"] if i["surface"] == "litellm-switchboard")
    assert "model_name: ods/current" in switchboard["content"]
    assert "model_name: local" in switchboard["content"]
    assert "model_name: default" in switchboard["content"]
    assert 'model_name: "*"' in switchboard["content"]
    assert "api_base: http://model-router:9099/v1" in switchboard["content"]
    assert switchboard["content"].count("http://model-router:9099/v1") == 4


def test_switchboard_route_injects_no_thinking_settings() -> None:
    # Any-model WP4.5: a model's thinking control reaches only the consumer that
    # sends per-request kwargs (Pixel's rendered config). The switchboard route
    # must pass requests through unchanged, whatever model is active.
    enabled = run_renderer("--surface", "all", "--switchboard-mode", "enabled")
    switchboard = file_by_surface(enabled, "litellm-switchboard")["content"]
    shipped = (ROOT / "config" / "litellm" / "switchboard.yaml").read_text(encoding="utf-8")
    for content in (switchboard, shipped):
        assert "enable_thinking" not in content
        assert "chat_template_kwargs" not in content
        assert "reasoning_effort" not in content


def test_local_profiles_allow_long_agent_streams() -> None:
    local = run_renderer("--surface", "litellm-local")
    local_config = file_by_surface(local, "litellm-local")["content"]
    assert "request_timeout: 900" in local_config
    assert "stream_timeout: 900" in local_config
    assert local_config.count("enable_thinking: false") == 3

    hybrid = run_renderer("--surface", "litellm-hybrid", "--ods-mode", "hybrid")
    hybrid_config = file_by_surface(hybrid, "litellm-hybrid")["content"]
    assert "request_timeout: 900" in hybrid_config
    assert "stream_timeout: 900" in hybrid_config
    assert hybrid_config.count("enable_thinking: false") == 3


def test_all_selects_one_mode_config() -> None:
    expected = {
        "local": "litellm-local",
        "cloud": "litellm-cloud",
        "hybrid": "litellm-hybrid",
        # One release: the retired managed-AMD mode renders exactly as local.
        "lemonade": "litellm-local",
    }
    all_mode_surfaces = set(expected.values())
    for mode, expected_surface in expected.items():
        payload = run_renderer("--surface", "all", "--ods-mode", mode)
        surfaces = {item["surface"] for item in payload["files"]}
        assert surfaces & all_mode_surfaces == {expected_surface}
        assert "litellm-lemonade" not in surfaces
        mode_config = file_by_surface(payload, expected_surface)["content"]
        assert "model_name: ods/current" in mode_config
        env_content = file_by_surface(payload, "env")["content"]
        assert f"ODS_MODE={'local' if mode == 'lemonade' else mode}\n" in env_content


def test_cloud_enabled_never_renders_local_switchboard() -> None:
    payload = run_renderer(
        "--surface",
        "all",
        "--ods-mode",
        "cloud",
        "--switchboard-mode",
        "enabled",
    )
    surfaces = {item["surface"] for item in payload["files"]}
    assert "litellm-cloud" in surfaces
    assert "litellm-switchboard" not in surfaces
    cloud = file_by_surface(payload, "litellm-cloud")["content"]
    assert "model_name: ods/current" in cloud
    assert "model-router" not in cloud


def test_remote_cloud_projection_uses_internal_egress_and_state_receipt() -> None:
    payload = run_renderer(
        "--surface",
        "all",
        "--ods-mode",
        "cloud",
        "--remote-llm-enabled",
        "true",
        "--remote-llm-transport",
        "direct",
        "--remote-llm-base-url",
        "https://gpu.example.test",
        "--remote-llm-model",
        "qwen/remote:latest",
    )
    surfaces = {item["surface"] for item in payload["files"]}
    assert "litellm-cloud" in surfaces
    assert "remote-routing-state" in surfaces
    assert "litellm-switchboard" not in surfaces

    cloud = file_by_surface(payload, "litellm-cloud")["content"]
    assert "model_name: ods/current" in cloud
    assert 'model: "openai/qwen/remote:latest"' in cloud
    assert 'model_name: "qwen/remote:latest"' in cloud
    assert 'api_base: "http://remote-provider-egress:8091/v1"' in cloud
    # The egress admits only the LiteLLM gateway key (GHSA-4rpc); the value
    # stays in LiteLLM's environment, never in the rendered file.
    assert cloud.count("api_key: os.environ/LITELLM_MASTER_KEY") == 3
    assert "not-needed" not in cloud
    assert "https://gpu.example.test" not in cloud
    assert "REMOTE_LLM_API_KEY" not in cloud

    env_content = file_by_surface(payload, "env")["content"]
    assert "REMOTE_LLM_ENABLED=true" in env_content
    assert "REMOTE_LLM_TRANSPORT=direct" in env_content
    assert "REMOTE_LLM_BASE_URL=https://gpu.example.test/v1" in env_content
    assert "REMOTE_LLM_MODEL=qwen/remote:latest" in env_content

    state = json.loads(file_by_surface(payload, "remote-routing-state")["content"])
    assert state["schema"] == "ods.remote-routing-state.v1"
    assert state["enabled"] is True
    assert state["mode"] == "cloud"
    assert state["provider"] == {
        "baseUrl": "https://gpu.example.test/v1",
        "capability": "openai-compatible",
        "model": "qwen/remote:latest",
        "transport": "direct",
    }
    assert state["projection"] == {
        "consumerRoute": "gateway",
        "egressBaseUrl": "http://remote-provider-egress:8091/v1",
        "gateway": "litellm-cloud",
        "publicModel": "ods/current",
    }
    assert state["status"] == {
        "proven": False,
        "reason": "pending-provider-handshake",
    }
    assert "key" not in json.dumps(state).lower()


def test_remote_routing_state_disabled_receipt_has_no_provider() -> None:
    payload = run_renderer("--surface", "remote-routing-state")
    state = json.loads(file_by_surface(payload, "remote-routing-state")["content"])
    assert state["enabled"] is False
    assert state["provider"] is None
    assert state["status"] == {"proven": False, "reason": "disabled"}
    assert "REMOTE_LLM_API_KEY" not in json.dumps(state)


def test_remote_projection_requires_cloud_mode() -> None:
    proc = subprocess.run(
        [
            sys.executable,
            str(SCRIPT),
            "--surface",
            "all",
            "--ods-mode",
            "local",
            "--remote-llm-enabled",
            "true",
            "--remote-llm-transport",
            "direct",
            "--remote-llm-base-url",
            "https://gpu.example.test/v1",
            "--remote-llm-model",
            "qwen-remote",
        ],
        cwd=ROOT,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    assert proc.returncode == 2
    assert "ODS_MODE=cloud" in proc.stderr


def test_remote_projection_rejects_unsafe_model_id() -> None:
    proc = subprocess.run(
        [
            sys.executable,
            str(SCRIPT),
            "--surface",
            "all",
            "--ods-mode",
            "cloud",
            "--remote-llm-enabled",
            "true",
            "--remote-llm-transport",
            "direct",
            "--remote-llm-base-url",
            "https://gpu.example.test/v1",
            "--remote-llm-model",
            "bad model; touch nope",
        ],
        cwd=ROOT,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    assert proc.returncode == 2
    assert "model id without spaces" in proc.stderr


def test_explicit_cloud_switchboard_render_fails_closed() -> None:
    proc = subprocess.run(
        [
            sys.executable,
            str(SCRIPT),
            "--surface",
            "litellm-switchboard",
            "--ods-mode",
            "cloud",
        ],
        cwd=ROOT,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    assert proc.returncode == 2
    assert "local-runtime-only" in proc.stderr


def test_native_local_projection_uses_host_route_and_concrete_model() -> None:
    payload = run_renderer(
        "--surface",
        "litellm-local-native",
        "--ods-mode",
        "local",
        "--gguf-file",
        "Native-Model.gguf",
        "--llm-base-url",
        "http://host.docker.internal:13306/v1",
    )
    content = file_by_surface(payload, "litellm-local-native")["content"]
    assert "model_name: ods/current" in content
    assert "model: openai/Native-Model.gguf" in content
    assert "api_base: http://host.docker.internal:13306/v1" in content
    assert "enable_thinking: false" in content
    assert "request_timeout: 900" in content
    assert "stream_timeout: 900" in content
    # An unkeyed host-native server keeps the placeholder key.
    assert content.count("api_key: not-needed") == 3


def test_native_local_projection_references_the_runtime_key_by_name() -> None:
    secret = "5e" * 32
    env = os.environ.copy()
    env["LLAMA_SERVER_API_KEY"] = secret
    proc = subprocess.run(
        [sys.executable, str(SCRIPT), "--surface", "litellm-local-native", "--gguf-file", "Native-Model.gguf",
         "--llm-base-url", "http://host.docker.internal:18080/v1", "--llm-api-key-env", "LLAMA_SERVER_API_KEY"],
        cwd=ROOT, env=env, text=True, capture_output=True, check=True,
    )
    content = file_by_surface(json.loads(proc.stdout), "litellm-local-native")["content"]
    assert content.count("api_key: os.environ/LLAMA_SERVER_API_KEY") == 3
    assert "not-needed" not in content
    assert secret not in proc.stdout + proc.stderr


def test_api_key_env_must_be_a_variable_name() -> None:
    for name in ("llama_server_api_key", "KEY=value", "$(touch x)", "5E" * 40):
        proc = subprocess.run(
            [sys.executable, str(SCRIPT), "--surface", "model-router-endpoints", "--llm-api-key-env", name],
            cwd=ROOT, text=True, capture_output=True,
        )
        assert proc.returncode != 0, name
        assert "environment variable name" in proc.stderr


def test_keyed_host_native_consumers_use_the_gateway() -> None:
    payload = run_renderer(
        "--surface", "all", "--switchboard-mode", "observe", "--gpu-backend", "amd",
        "--gguf-file", "Native-Model.gguf", "--model", "native-model",
        "--llm-base-url", "http://host.docker.internal:18080/v1",
        "--llm-api-key-env", "LLAMA_SERVER_API_KEY", "--litellm-key", "sk-gateway",
    )
    # Consumers cannot hold the runtime key; they reach it through LiteLLM.
    opencode = json.loads(file_by_surface(payload, "opencode")["content"])
    assert opencode["baseURL"] == "http://litellm:4000/v1"
    assert opencode["apiKey"] == "sk-gateway"
    perplexica = json.loads(file_by_surface(payload, "perplexica")["content"])
    openai_provider = model_provider_by_id(perplexica, "openai")
    assert openai_provider["config"]["baseURL"] == "http://litellm:4000/v1"
    assert perplexica["preferences"]["defaultChatModel"] == "Native-Model.gguf"
    endpoints = json.loads(file_by_surface(payload, "model-router-endpoints")["content"])
    assert endpoints == {"endpoints": [{"id": "llama-server-default",
                                        "baseUrl": "http://host.docker.internal:18080",
                                        "apiKeyEnv": "LLAMA_SERVER_API_KEY"}]}


def test_checked_in_mode_configs_match_renderer() -> None:
    for mode in ("local", "cloud", "hybrid"):
        payload = run_renderer("--surface", f"litellm-{mode}", "--ods-mode", mode)
        rendered = file_by_surface(payload, f"litellm-{mode}")["content"]
        checked_in = (ROOT / "config" / "litellm" / f"{mode}.yaml").read_text(
            encoding="utf-8"
        )
        assert rendered == checked_in


def test_enabled_env_exports_switchboard_webui_gateway() -> None:
    payload = run_renderer(
        "--surface",
        "env",
        "--switchboard-mode",
        "enabled",
        "--litellm-key",
        "sk-test-litellm",
    )
    content = file_by_surface(payload, "env")["content"]
    assert "ODS_MODEL_SWITCHBOARD=enabled" in content
    assert "OPEN_WEBUI_LLM_BASE_URL=http://litellm:4000" in content
    assert "OPEN_WEBUI_LLM_API_KEY=sk-test-litellm" in content


def test_enabled_perplexica_uses_stable_alias() -> None:
    payload = run_renderer(
        "--surface",
        "perplexica",
        "--switchboard-mode",
        "enabled",
        "--ods-mode",
        "lemonade",
        "--gpu-backend",
        "amd",
        "--gguf-file",
        "Concrete.gguf",
        "--litellm-key",
        "sk-test-litellm",
    )
    content = json.loads(file_by_surface(payload, "perplexica")["content"])
    openai_provider = model_provider_by_id(content, "openai")
    assert content["preferences"]["defaultChatModel"] == "ods/current"
    assert openai_provider["config"]["baseURL"] == "http://litellm:4000/v1"
    assert openai_provider["config"]["apiKey"] == "sk-test-litellm"
    assert openai_provider["chatModels"][0]["key"] == "ods/current"


def test_enabled_hermes_uses_stable_switchboard_alias() -> None:
    payload = run_renderer(
        "--surface",
        "hermes",
        "--switchboard-mode",
        "enabled",
        "--gguf-file",
        "Raw-Runtime.gguf",
        "--llm-base-url",
        "http://llama-server:8080/v1",
    )
    content = file_by_surface(payload, "hermes")["content"]
    assert 'default: "ods/current"' in content
    assert "Raw-Runtime.gguf" not in content
    assert 'base_url: "http://litellm:4000/v1"' in content


def test_enabled_opencode_uses_stable_switchboard_alias() -> None:
    payload = run_renderer(
        "--surface",
        "opencode",
        "--switchboard-mode",
        "enabled",
        "--litellm-key",
        "switch-secret",
    )
    content = json.loads(file_by_surface(payload, "opencode")["content"])
    assert content["model"] == "ods/current"
    assert content["baseURL"] == "http://litellm:4000/v1"
    assert content["apiKey"] == "switch-secret"


def test_router_endpoints_strip_trailing_v1() -> None:
    payload = run_renderer(
        "--surface", "model-router-endpoints",
        "--llm-base-url", "http://llama-server:8080/v1",
        "--gpu-backend", "amd",
        # Retired flag, accepted and ignored for one release (R13).
        "--lemonade-api-base", "http://lemonade:8000/api/v1",
    )
    content = json.loads(payload["files"][0]["content"])
    # One managed runtime, one endpoint: no lemonade-default row on AMD.
    assert content == {"endpoints": [{"id": "llama-server-default", "baseUrl": "http://llama-server:8080"}]}


def test_llama_server_backends_render_todays_bytes() -> None:
    baseline = run_renderer(*ALL_ENABLED, env=llm_backend_env())
    assert baseline["inputs"]["llm_backend"] == "llama-server"
    assert file_by_surface(baseline, "model-router-endpoints")["content"] == LLAMA_SERVER_ENDPOINTS
    assert file_by_surface(baseline, "env")["content"] == LLAMA_SERVER_ENV
    # Every other .env LLM_BACKEND value, by flag or environment, renders
    # every surface exactly as before.
    for value in ("", "llama-server", "external", "hipfire", "lemonade", "litellm"):
        assert run_renderer(*ALL_ENABLED, env=llm_backend_env(value))["files"] == baseline["files"], value
        if value:
            flagged = run_renderer(*ALL_ENABLED, "--llm-backend", value, env=llm_backend_env())
            assert flagged["files"] == baseline["files"], value


# The vllm endpoint carries the served model's thinking map (catalog
# reasoning + thinking_template_kwargs): model-router translates a request's
# level with it, since GLM-5.3 and Qwen3.8 read different template arguments.
VLLM_MODEL = ("--model", "glm-5.3-flash-nvfp4")
GLM_THINKING = {"default": "medium", "templateKwargs": {
    "off": {"reasoning_effort": "low"}, "low": {"reasoning_effort": "low"},
    "medium": {"reasoning_effort": "high"}, "high": {}}}
QWEN_THINKING = {"default": "medium", "templateKwargs": {
    "off": {"enable_thinking": False}, "low": {"reasoning_effort": "low"},
    "medium": {"reasoning_effort": "medium"}, "high": {"reasoning_effort": "xhigh"}}}


def test_vllm_backend_routes_model_router_to_vllm_server() -> None:
    surface = ("--surface", "model-router-endpoints", *VLLM_MODEL)
    for payload in (
        run_renderer(*surface, "--llm-backend", "vllm", env=llm_backend_env()),
        run_renderer(*surface, env=llm_backend_env("vllm")),
        # The llama-server origin and key inputs do not describe vllm-server.
        run_renderer(*surface, "--llm-backend", "vllm",
                     "--llm-base-url", "http://host.docker.internal:18080/v1",
                     "--llm-api-key-env", "LLAMA_SERVER_API_KEY", env=llm_backend_env()),
    ):
        assert payload["inputs"]["llm_backend"] == "vllm"
        content = file_by_surface(payload, "model-router-endpoints")["content"]
        assert json.loads(content) == {
            "endpoints": [{"id": "vllm-default", "baseUrl": "http://vllm-server:8000",
                           "thinking": GLM_THINKING}],
        }


def test_vllm_endpoint_carries_each_models_own_thinking_map() -> None:
    surface = ("--surface", "model-router-endpoints", "--llm-backend", "vllm")
    for model, thinking in (("glm-5.3-flash-nvfp4", GLM_THINKING),
                            ("qwen3.8-flash-next-nvfp4", QWEN_THINKING),
                            # The catalog id names the same entry.
                            ("qwen3.8-flash-next-nvfp4-vllm", QWEN_THINKING)):
        payload = run_renderer(*surface, "--model", model, env=llm_backend_env())
        endpoint = json.loads(file_by_surface(payload, "model-router-endpoints")["content"])["endpoints"][0]
        assert endpoint["thinking"] == thinking, model


def test_vllm_endpoint_refuses_a_model_without_a_vllm_catalog_entry() -> None:
    for model in ("qwen3.5-9b", "no-such-model"):
        proc = subprocess.run(
            [sys.executable, str(SCRIPT), "--surface", "model-router-endpoints",
             "--llm-backend", "vllm", "--model", model],
            cwd=ROOT, env=llm_backend_env(), text=True, capture_output=True,
        )
        assert proc.returncode == 2, model
        assert "needs --model to name one vLLM catalog entry" in proc.stderr, proc.stderr
    # Only the router endpoint needs the entry; other surfaces render as before.
    switchboard = run_renderer("--surface", "litellm-switchboard", "--llm-backend", "vllm",
                               "--litellm-key", "sk-test-litellm", env=llm_backend_env())
    assert switchboard["inputs"]["vllm_thinking"] is None


def test_vllm_backend_changes_only_the_router_endpoint_and_env_backend() -> None:
    baseline = run_renderer(*ALL_ENABLED, *VLLM_MODEL, env=llm_backend_env())
    vllm = run_renderer(*ALL_ENABLED, *VLLM_MODEL, "--llm-backend", "vllm", env=llm_backend_env())
    assert "LLM_BACKEND=vllm\n" in file_by_surface(vllm, "env")["content"]
    assert file_by_surface(vllm, "env")["content"] == file_by_surface(baseline, "env")["content"].replace(
        "LLM_BACKEND=llama-server\n", "LLM_BACKEND=vllm\n")
    changed = {after["surface"] for after, before in zip(vllm["files"], baseline["files"])
               if after != before}
    assert changed == {"env", "model-router-endpoints"}


def test_retired_lemonade_flags_are_accepted_and_ignored() -> None:
    payload = run_renderer(
        "--surface",
        "all",
        "--switchboard-mode",
        "observe",
        "--ods-mode",
        "lemonade",
        "--gpu-backend",
        "amd",
        "--gguf-file",
        "Modern-Model.gguf",
        "--model",
        "modern-model",
        "--lemonade-model-id",
        "extra.Modern-Model.gguf",
        "--lemonade-api-base",
        "http://host.docker.internal:13305/api/v1",
    )

    surfaces = {item["surface"] for item in payload["files"]}
    env_content = file_by_surface(payload, "env")["content"]
    hermes_content = file_by_surface(payload, "hermes")["content"]
    opencode = json.loads(file_by_surface(payload, "opencode")["content"])
    perplexica = json.loads(file_by_surface(payload, "perplexica")["content"])

    assert "litellm-lemonade" not in surfaces and "litellm-local" in surfaces
    assert "ODS_MODE=local\n" in env_content
    assert "LLM_BACKEND=llama-server\n" in env_content
    assert "LEMONADE" not in env_content
    assert 'default: "Modern-Model.gguf"' in hermes_content
    assert opencode["model"] == "modern-model"
    assert perplexica["preferences"]["defaultChatModel"] == "Modern-Model.gguf"
    assert "extra." not in json.dumps(payload)
    assert "13305" not in json.dumps(payload)


def test_litellm_lemonade_surface_is_retired() -> None:
    proc = subprocess.run(
        [sys.executable, str(SCRIPT), "--surface", "litellm-lemonade"],
        cwd=ROOT, text=True, capture_output=True,
    )
    assert proc.returncode == 2
    assert "litellm-lemonade" in proc.stderr


def test_amd_local_env_has_no_lemonade_keys() -> None:
    payload = run_renderer(
        "--surface",
        "env",
        "--ods-mode",
        "local",
        "--gpu-backend",
        "amd",
        "--gguf-file",
        "Fallback-Model.gguf",
    )

    env_content = file_by_surface(payload, "env")["content"]
    assert "LEMONADE" not in env_content
    assert "LLM_BACKEND=llama-server\n" in env_content
    assert "GGUF_FILE=Fallback-Model.gguf\n" in env_content


def test_hermes_uses_the_gguf_alias_for_amd() -> None:
    payload = run_renderer(
        "--surface",
        "hermes",
        "--switchboard-mode",
        "observe",
        "--ods-mode",
        "local",
        "--gpu-backend",
        "amd",
        "--gguf-file",
        "Amd.gguf",
        "--llm-base-url",
        "http://litellm:4000/v1",
        "--context-length",
        "65536",
    )
    content = file_by_surface(payload, "hermes")["content"]
    assert 'default: "Amd.gguf"' in content
    assert 'base_url: "http://litellm:4000/v1"' in content
    assert "context_length: 65536" in content
    assert "max_tokens: 1024" in content


def test_perplexica_default_model_matches_route() -> None:
    payload = run_renderer(
        "--surface",
        "perplexica",
        "--switchboard-mode",
        "observe",
        "--ods-mode",
        "local",
        "--gpu-backend",
        "amd",
        "--gguf-file",
        "Research.gguf",
    )
    content = json.loads(file_by_surface(payload, "perplexica")["content"])
    openai_provider = model_provider_by_id(content, "openai")
    assert content["preferences"]["defaultChatModel"] == "Research.gguf"
    assert openai_provider["chatModels"][0]["name"] == "Research.gguf"


def test_write_mode_writes_under_output_root() -> None:
    with tempfile.TemporaryDirectory() as tmp:
        proc = subprocess.run(
            [
                sys.executable,
                str(SCRIPT),
                "--surface",
                "litellm-local-native",
                "--gpu-backend",
                "amd",
                "--gguf-file",
                "Written.gguf",
                "--output-root",
                tmp,
                "--write",
                "--format",
                "json",
            ],
            cwd=ROOT,
            text=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            check=True,
        )
        payload = json.loads(proc.stdout)
        target = Path(tmp) / "config" / "litellm" / "local.yaml"
        assert payload["mode"] == "write"
        assert target.exists()
        assert "openai/Written.gguf" in target.read_text(encoding="utf-8")
        if os.name != "nt":
            assert target.stat().st_mode & 0o777 == 0o644
        assert not list(target.parent.glob(f".{target.name}.*.tmp"))


def test_model_router_allowlist_mount_is_readable_after_private_source_staging() -> None:
    if os.name == "nt":
        return  # POSIX host modes are the contract Docker bind mounts preserve.
    with tempfile.TemporaryDirectory() as tmp:
        root = Path(tmp)
        endpoint_dir = root / "config" / "model-router"
        endpoint_dir.mkdir(parents=True)
        endpoint = endpoint_dir / "endpoints.json"
        endpoint.write_text('{"endpoints": []}\n', encoding="utf-8")
        endpoint_dir.chmod(0o700)
        endpoint.chmod(0o600)
        private_dir = root / "config" / "private"
        private_dir.mkdir()
        private_file = private_dir / "key.txt"
        private_file.write_text("private\n", encoding="utf-8")
        private_dir.chmod(0o700)
        private_file.chmod(0o600)

        for _ in range(2):  # Activation re-renders the same mounted path.
            run_renderer("--surface", "model-router-endpoints", "--output-root",
                         tmp, "--write", "--format", "json")
            assert endpoint_dir.stat().st_mode & 0o777 == 0o711
            assert endpoint.stat().st_mode & 0o777 == 0o644
            assert json.loads(endpoint.read_text(encoding="utf-8"))["endpoints"][0]["id"] == "llama-server-default"
            assert private_dir.stat().st_mode & 0o777 == 0o700
            assert private_file.stat().st_mode & 0o777 == 0o600
            endpoint_dir.chmod(0o700)
            endpoint.chmod(0o600)


def test_model_router_allowlist_rejects_embedded_credentials_before_public_write() -> None:
    with tempfile.TemporaryDirectory() as tmp:
        proc = subprocess.run(
            [sys.executable, str(SCRIPT), "--surface", "model-router-endpoints",
             "--llm-base-url", "http://user:secret@localhost:8080/v1",
             "--output-root", tmp, "--write"],
            cwd=ROOT, text=True, capture_output=True,
        )
        assert proc.returncode != 0
        assert "secret" not in proc.stderr
        assert not (Path(tmp) / "config" / "model-router" / "endpoints.json").exists()


def test_model_router_allowlist_does_not_widen_symlinked_directory() -> None:
    if os.name == "nt":
        return
    with tempfile.TemporaryDirectory() as tmp:
        root = Path(tmp)
        private = root / "private"
        private.mkdir(mode=0o700)
        config = root / "config"
        config.mkdir()
        (config / "model-router").symlink_to(private, target_is_directory=True)
        proc = subprocess.run(
            [sys.executable, str(SCRIPT), "--surface", "model-router-endpoints",
             "--output-root", tmp, "--write"], cwd=ROOT, text=True,
            capture_output=True,
        )
        assert proc.returncode != 0
        assert private.stat().st_mode & 0o777 == 0o700
        assert not (private / "endpoints.json").exists()


def test_write_cli_defaults_to_secret_free_paths() -> None:
    secret = "renderer-secret-must-not-reach-output"
    with tempfile.TemporaryDirectory() as tmp:
        env = os.environ.copy()
        env["ODS_RENDER_LITELLM_KEY"] = secret
        proc = subprocess.run(
            [
                sys.executable,
                str(SCRIPT),
                "--surface",
                "opencode",
                "--switchboard-mode",
                "enabled",
                "--gpu-backend",
                "amd",
                "--gguf-file",
                "Private.gguf",
                "--output-root",
                tmp,
                "--write",
            ],
            cwd=ROOT,
            env=env,
            text=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            check=True,
        )
        combined = proc.stdout + proc.stderr
        target = Path(tmp) / ".opencode" / "auth.json"
        assert secret not in combined
        assert proc.stdout.strip() == ".opencode/auth.json"
        assert secret in target.read_text(encoding="utf-8")


def test_atomic_write_failure_preserves_known_good_config() -> None:
    renderer = load_renderer_module()
    with tempfile.TemporaryDirectory() as tmp:
        target = Path(tmp) / "local.yaml"
        target.write_text("known-good\n", encoding="utf-8")
        original_replace = renderer.os.replace
        original_sleep = renderer.time.sleep

        def fail_replace(*_args, **_kwargs) -> None:
            raise PermissionError("injected replace failure")

        renderer.os.replace = fail_replace
        renderer.time.sleep = lambda _seconds: None
        try:
            try:
                renderer.atomic_write_text(target, "new-route\n")
            except PermissionError as exc:
                assert "injected replace failure" in str(exc)
            else:
                raise AssertionError("fault injection did not fail the replace")
        finally:
            renderer.os.replace = original_replace
            renderer.time.sleep = original_sleep

        assert target.read_text(encoding="utf-8") == "known-good\n"
        assert not list(target.parent.glob(f".{target.name}.*.tmp"))


def test_identical_switchboard_roundtrip_preserves_bound_inode() -> None:
    with tempfile.TemporaryDirectory() as tmp:
        target = Path(tmp) / "config/litellm/switchboard.yaml"
        def render(model):
            subprocess.run([sys.executable, str(SCRIPT), "--surface", "litellm-switchboard",
                            "--model", model, "--output-root", tmp, "--write"],
                           check=True, stdout=subprocess.DEVNULL)
        render("qwen3.5-9b")
        before = target.stat()
        with target.open("rb") as bound:
            content = bound.read()
            render("qwen3.5-2b")
            render("qwen3.5-9b")
            after = target.stat()
            assert (before.st_dev, before.st_ino, before.st_mtime_ns) == (after.st_dev, after.st_ino, after.st_mtime_ns)
            assert target.read_bytes() == content
            assert os.fstat(bound.fileno()).st_nlink == 1


def test_changed_bytes_and_modes_still_replace_atomically() -> None:
    renderer = load_renderer_module()
    with tempfile.TemporaryDirectory() as tmp:
        target = Path(tmp) / "config"
        renderer.atomic_write_text(target, "old\n")
        with target.open("rb") as bound:
            renderer.atomic_write_text(target, "new\n")
            assert bound.read() == b"old\n"
            assert target.read_bytes() == b"new\n"
        if os.name != "nt":
            target.chmod(0o600)
            renderer.atomic_write_text(target, "new\n", file_mode=0o644)
            assert target.stat().st_mode & 0o777 == 0o644


def test_identical_symlink_and_hardlink_do_not_skip_replacement() -> None:
    if os.name == "nt":
        return
    renderer = load_renderer_module()
    with tempfile.TemporaryDirectory() as tmp:
        original = Path(tmp) / "original"
        original.write_text("same\n")
        target = Path(tmp) / "target"
        target.symlink_to(original)
        renderer.atomic_write_text(target, "same\n")
        assert not target.is_symlink()
        target.unlink()
        os.link(original, target)
        renderer.atomic_write_text(target, "same\n")
        assert original.stat().st_ino != target.stat().st_ino
        assert original.read_text() == "same\n"


def test_validation_rejects_negative_context_or_invalid_port() -> None:
    proc = subprocess.run(
        [sys.executable, str(SCRIPT), "--context-length", "-10"],
        cwd=ROOT,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    assert proc.returncode != 0
    assert "context length must be positive" in proc.stderr

    proc = subprocess.run(
        [sys.executable, str(SCRIPT), "--opencode-port", "70000"],
        cwd=ROOT,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    assert proc.returncode != 0
    assert "opencode port must be between 1 and 65535" in proc.stderr


def test_validation_rejects_control_characters_in_model_and_key() -> None:
    proc = subprocess.run(
        [sys.executable, str(SCRIPT), "--model", "qwen\ninjected=true"],
        cwd=ROOT,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    assert proc.returncode != 0
    assert "cannot contain control characters or newlines" in proc.stderr


def main() -> int:
    tests = [
        test_external_model_uses_authenticated_gateway_without_vendor_impersonation,
        test_external_gateway_rejects_credentialed_or_malformed_bases,
        test_all_surfaces_render,
        test_switchboard_surface_gated_on_enabled_mode,
        test_local_profiles_allow_long_agent_streams,
        test_all_selects_one_mode_config,
        test_cloud_enabled_never_renders_local_switchboard,
        test_remote_cloud_projection_uses_internal_egress_and_state_receipt,
        test_remote_routing_state_disabled_receipt_has_no_provider,
        test_remote_projection_requires_cloud_mode,
        test_remote_projection_rejects_unsafe_model_id,
        test_explicit_cloud_switchboard_render_fails_closed,
        test_native_local_projection_uses_host_route_and_concrete_model,
        test_native_local_projection_references_the_runtime_key_by_name,
        test_api_key_env_must_be_a_variable_name,
        test_keyed_host_native_consumers_use_the_gateway,
        test_checked_in_mode_configs_match_renderer,
        test_enabled_env_exports_switchboard_webui_gateway,
        test_enabled_perplexica_uses_stable_alias,
        test_enabled_hermes_uses_stable_switchboard_alias,
        test_enabled_opencode_uses_stable_switchboard_alias,
        test_router_endpoints_strip_trailing_v1,
        test_llama_server_backends_render_todays_bytes,
        test_vllm_backend_routes_model_router_to_vllm_server,
        test_vllm_endpoint_carries_each_models_own_thinking_map,
        test_vllm_endpoint_refuses_a_model_without_a_vllm_catalog_entry,
        test_vllm_backend_changes_only_the_router_endpoint_and_env_backend,
        test_retired_lemonade_flags_are_accepted_and_ignored,
        test_litellm_lemonade_surface_is_retired,
        test_amd_local_env_has_no_lemonade_keys,
        test_hermes_uses_the_gguf_alias_for_amd,
        test_perplexica_default_model_matches_route,
        test_write_mode_writes_under_output_root,
        test_write_cli_defaults_to_secret_free_paths,
        test_atomic_write_failure_preserves_known_good_config,
        test_identical_switchboard_roundtrip_preserves_bound_inode,
        test_changed_bytes_and_modes_still_replace_atomically,
        test_identical_symlink_and_hardlink_do_not_skip_replacement,
        test_validation_rejects_negative_context_or_invalid_port,
        test_validation_rejects_control_characters_in_model_and_key,
    ]
    for test in tests:
        test()
        print(f"[PASS] {test.__name__}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())