Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

4.0 KB
Raw Blame
"""Resolve setup inference and destination-bound credentials from one snapshot."""
from urllib.parse import urlsplit, urlunsplit

from config import read_live_env_values

KEYS = (
    "LLM_API_URL", "OLLAMA_URL", "LLM_API_BASE_PATH", "LLM_MODEL", "LLM_BACKEND",
    "GGUF_FILE", "ODS_MODEL_SWITCHBOARD", "LITELLM_KEY", "LITELLM_MASTER_KEY",
    "OPEN_WEBUI_LLM_BASE_URL", "OPEN_WEBUI_LLM_API_KEY", "OPEN_WEBUI_TASK_MODEL",
    "AMD_INFERENCE_LOCATION", "VLLM_SERVED_MODEL",
)
LLAMA_SERVER_HOSTS = ("llama-server", "ods-llama-server")
VLLM_HOSTS = ("vllm-server", "ods-vllm-server")


def api_base(raw: str, path: str = "/v1") -> str:
    raw = raw.strip().rstrip("/")
    if "\\" in raw or any(ord(c) < 32 for c in raw + path):
        raise ValueError("invalid inference URL")
    parsed = urlsplit(raw)
    if (parsed.scheme not in ("http", "https") or not parsed.hostname
            or parsed.username is not None or parsed.password is not None
            or parsed.query or parsed.fragment or parsed.port == 0):
        raise ValueError("invalid inference URL")
    path = "/" + (path.strip() or "/v1").strip("/")
    if any(c in path for c in "?#\\") or ".." in path.split("/"):
        raise ValueError("invalid inference API path")
    current = parsed.path.rstrip("/")
    if not current.endswith(("/v1", path)):
        current += path.rstrip("/")
    return urlunsplit((parsed.scheme, parsed.netloc, current, "", ""))


def same_endpoint(left: str, right: str) -> bool:
    def identity(url):
        value = urlsplit(url)
        return value.scheme, value.hostname, value.port or (443 if value.scheme == "https" else 80), value.path
    return identity(left) == identity(right)


def resolve_chat_route(default_url: str) -> tuple[str, str, dict[str, str]]:
    env = {key: value.strip() for key, value in read_live_env_values(KEYS).items()}
    backend = env["LLM_BACKEND"].lower()
    raw = env["LLM_API_URL"] or env["OLLAMA_URL"] or default_url
    if (env["AMD_INFERENCE_LOCATION"].lower() == "host" and backend != "external"
            and urlsplit(raw).hostname not in ("litellm", "ods-litellm")):
        # A Windows-hosted llama-server requires its key, which only LiteLLM
        # holds; setup chat reaches it through the gateway.
        raw = "http://litellm:4000"
    if backend == "vllm" and urlsplit(raw).hostname in LLAMA_SERVER_HOSTS:
        # LLM_BACKEND=vllm turns the in-stack llama-server off; the vLLM
        # engine answers through the gateway (LiteLLM, then model-router).
        raw = "http://litellm:4000"
    base = api_base(raw, env["LLM_API_BASE_PATH"] or "/v1")
    target = urlsplit(base)
    local_gateway = target.scheme == "http" and target.hostname in ("litellm", "ods-litellm") and target.port == 4000
    if local_gateway:
        # LiteLLM's public API path is independent of its backend's path.
        base = urlunsplit((target.scheme, target.netloc, "/v1", "", ""))
    paired_webui = False
    if env["OPEN_WEBUI_LLM_BASE_URL"]:
        paired_webui = same_endpoint(base, api_base(env["OPEN_WEBUI_LLM_BASE_URL"]))
    key = ""
    model = env["LLM_MODEL"] or "qwen3-coder-next"
    if local_gateway:
        key = env["LITELLM_KEY"] or env["LITELLM_MASTER_KEY"]
        model = (env["OPEN_WEBUI_TASK_MODEL"] if paired_webui else "") or (
            "ods/current" if env["ODS_MODEL_SWITCHBOARD"] == "enabled" or backend == "external" else "default")
    elif paired_webui:
        key = env["OPEN_WEBUI_LLM_API_KEY"]
        model = env["OPEN_WEBUI_TASK_MODEL"] or model
    elif target.hostname in VLLM_HOSTS and backend == "vllm":
        # vLLM serves exactly --served-model-name and refuses any other id.
        model = env["VLLM_SERVED_MODEL"] or model
    elif target.hostname in (*LLAMA_SERVER_HOSTS, "host.docker.internal") and backend != "external":
        # Every managed llama-server serves its model as --alias <GGUF_FILE>.
        model = env["GGUF_FILE"] or model
    headers = {"Content-Type": "application/json"}
    if key:
        headers["Authorization"] = f"Bearer {key}"
    return base + "/chat/completions", model, headers