Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

8.3 KB
Raw Blame
#!/bin/bash
# External Ollama / LM Studio discovery and validation helpers.

# A retained route must be decoded with the same Compose-compatible grammar
# used when .env is loaded for the rest of the installer.
# shellcheck source=../../lib/safe-env.sh
source "$(dirname "${BASH_SOURCE[0]}")/../../lib/safe-env.sh"

external_llm_normalize_model_name() {
    local value="${1:-}"
    value="${value##*/}"
    value="${value,,}"
    value="${value%.gguf}"
    value="$(printf '%s' "$value" | sed -E \
        -e 's/[-_.](q[0-9]+([_.][a-z0-9]+)*|iq[0-9]+([_.][a-z0-9]+)*|fp(8|16|32)|bf16)([-_.].*)?$//' \
        -e 's/:/-/' \
        -e 's/[[:space:]_]+/-/g' \
        -e 's/-+/-/g' \
        -e 's/^-|-$//g')"
    printf '%s\n' "$value"
}

external_llm_model_matches() {
    local expected actual
    expected="$(external_llm_normalize_model_name "${1:-}")"
    actual="$(external_llm_normalize_model_name "${2:-}")"
    [[ -n "$expected" && "$expected" == "$actual" ]]
}

external_llm_find_matching_model() {
    local target="${1:-}" candidate
    while IFS= read -r candidate; do
        if [[ -n "$candidate" ]] && external_llm_model_matches "$target" "$candidate"; then
            printf '%s\n' "$candidate"
            return 0
        fi
    done
    return 1
}

external_llm_strip_url() {
    local url="${1:-}"
    url="${url%/}"
    case "$url" in
        */api/v1) url="${url%/api/v1}" ;;
        */v1) url="${url%/v1}" ;;
    esac
    printf '%s\n' "$url"
}

external_llm_validate_url() {
    local url="${1:-}"
    EXTERNAL_LLM_VALIDATE_URL="$url" python3 - <<'PY'
import os
import sys
from urllib.parse import urlsplit

try:
    parsed = urlsplit(os.environ["EXTERNAL_LLM_VALIDATE_URL"])
    _ = parsed.port
except (KeyError, ValueError):
    sys.exit(1)

if (
    parsed.scheme not in {"http", "https"}
    or not parsed.hostname
    or parsed.username is not None
    or parsed.password is not None
    or parsed.query
    or parsed.fragment
    or parsed.path.rstrip("/") not in {"", "/v1", "/api/v1"}
):
    sys.exit(1)
PY
}

external_llm_host_url() {
    local url
    url="$(external_llm_strip_url "${1:-}")"
    url="${url/host.docker.internal/127.0.0.1}"
    printf '%s\n' "$url"
}

external_llm_container_url() {
    local url
    url="$(external_llm_strip_url "${1:-}")"
    if [[ "$url" =~ ^(https?://)(localhost|127\.0\.0\.1|\[::1\])([:/].*|$) ]]; then
        url="${BASH_REMATCH[1]}host.docker.internal${BASH_REMATCH[3]}"
    fi
    printf '%s\n' "$url"
}

# Read a credential without putting it in an installer argument, log, or curl
# command line. The installed copy is mounted into LiteLLM read-only.
external_llm_read_api_key() {
    local path="${1:-}"
    EXTERNAL_LLM_KEY_PATH="$path" python3 - <<'PY'
import os
import stat
import sys

path = os.environ.get("EXTERNAL_LLM_KEY_PATH", "")
try:
    fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
    info = os.fstat(fd)
    if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid():
        raise ValueError()
    if stat.S_IMODE(info.st_mode) not in (0o600, 0o400) or not 0 < info.st_size <= 4096:
        raise ValueError()
    with os.fdopen(fd, "rb") as handle:
        value = handle.read()
    value = value.removesuffix(b"\n")
    if not value or any(byte < 33 or byte > 126 for byte in value):
        raise ValueError()
    sys.stdout.write(value.decode("ascii"))
except (OSError, ValueError):
    print("External LLM key file must be owner-owned, private, and contain one printable ASCII line.", file=sys.stderr)
    sys.exit(1)
PY
}

external_llm_curl() {
    local key_file="${EXTERNAL_LLM_API_KEY_FILE:-}" key
    if [[ -n "$key_file" ]]; then
        key="$(external_llm_read_api_key "$key_file")" || return 1
        curl -H @<(printf 'Authorization: Bearer %s\n' "$key") "$@"
    elif [[ -n "${EXTERNAL_LLM_API_KEY_VALUE:-}" ]]; then
        # A key from --external-llm-key-env or a retired flag; phase 06
        # stores it as the key file.
        curl -H @<(printf 'Authorization: Bearer %s\n' "$EXTERNAL_LLM_API_KEY_VALUE") "$@"
    else
        curl "$@"
    fi
}

external_llm_models() {
    local provider="${1:-}" url="${2:-}" response
    url="$(external_llm_host_url "$url")"
    case "$provider" in
        ollama)
            response="$(external_llm_curl -fsS --max-time 5 "${url}/api/tags" 2>/dev/null)" || return 1
            ;;
        lmstudio|openai-compatible)
            response="$(external_llm_curl -fsS --max-time 5 "${url}/v1/models" 2>/dev/null)" || return 1
            ;;
        *)
            return 2
            ;;
    esac

    python3 -c '
import json
import sys

provider = sys.argv[1]
payload = json.load(sys.stdin)
if provider == "ollama":
    values = (item.get("name") or item.get("model") for item in payload.get("models", []))
else:
    values = (item.get("id") for item in payload.get("data", []))
for value in values:
    if isinstance(value, str) and value:
        print(value)
' "$provider" <<<"$response"
}

external_llm_detect_provider() {
    local url="${1:-}"
    if external_llm_models ollama "$url" >/dev/null 2>&1; then
        printf 'ollama\n'
        return 0
    fi
    if external_llm_models lmstudio "$url" >/dev/null 2>&1; then
        # /v1/models proves the protocol, not the vendor's identity.
        printf 'openai-compatible\n'
        return 0
    fi
    return 1
}

external_llm_resolve_model() {
    local provider="${1:-}" url="${2:-}" requested="${3:-}" target="${4:-}"
    local models
    models="$(external_llm_models "$provider" "$url")" || return 1
    if [[ -n "$requested" ]]; then
        if grep -Fqx -- "$requested" <<<"$models"; then
            printf '%s\n' "$requested"
            return 0
        fi
        return 2
    fi
    external_llm_find_matching_model "$target" <<<"$models"
}

# Why the model list could not be read, for an installer message. Prints
# unreachable, key-required, key-refused, http-<status>, or reachable (the
# list answered, so the model itself is the problem). Called only after
# discovery has failed.
external_llm_diagnose() {
    local provider="${1:-}" url="${2:-}" path="/v1/models" status
    url="$(external_llm_host_url "$url")"
    [[ "$provider" != ollama ]] || path="/api/tags"
    status="$(external_llm_curl -sS -o /dev/null --max-time 5 -w '%{http_code}' "${url}${path}" 2>/dev/null)" || {
        printf 'unreachable\n'
        return 0
    }
    case "$status" in
        2[0-9][0-9]) printf 'reachable\n' ;;
        401|403)
            if [[ -n "${EXTERNAL_LLM_API_KEY_FILE:-}${EXTERNAL_LLM_API_KEY_VALUE:-}" ]]; then
                printf 'key-refused\n'
            else
                printf 'key-required\n'
            fi
            ;;
        [0-9][0-9][0-9]) printf 'http-%s\n' "$status" ;;
        *) printf 'unreachable\n' ;;
    esac
}

external_llm_probe_completion() {
    local url="${1:-}" model="${2:-}" body attempt curl_status
    url="$(external_llm_host_url "$url")"
    body="$(
        EXTERNAL_LLM_MODEL_VALUE="$model" python3 - <<'PY'
import json
import os

print(json.dumps({
    "model": os.environ["EXTERNAL_LLM_MODEL_VALUE"],
    "messages": [{"role": "user", "content": "Reply with OK."}],
    "max_tokens": 1,
    "temperature": 0,
    "stream": False,
}))
PY
    )"
    # A discovered external model can be serving another long prompt when the
    # installer makes its first real completion. Retry once after a transport
    # failure, but never accept discovery alone as proof of working inference.
    for attempt in 1 2; do
        if external_llm_curl -fsS --max-time "${EXTERNAL_LLM_PROBE_TIMEOUT:-60}" \
            -H "Content-Type: application/json" \
            -d "$body" \
            "${url}/v1/chat/completions" >/dev/null; then
            return 0
        else
            curl_status=$?
        fi
        printf 'External LLM completion probe attempt %d/2 failed (curl exit %d).\n' \
            "$attempt" "$curl_status" >&2
        # HTTP errors and malformed requests need a configuration fix, not a
        # second inference attempt. Only transient transport failures retry.
        case "$curl_status" in
            7|28|52|55|56) ;;
            *) return "$curl_status" ;;
        esac
        if [[ "$attempt" -eq 1 ]]; then
            sleep 2
        fi
    done
    return "$curl_status"
}

external_llm_env_value() {
    local env_file="${1:-}" key="${2:-}" value
    [[ -f "$env_file" ]] || return 1
    value="$(grep -m1 "^${key}=" "$env_file" 2>/dev/null | cut -d= -f2- || true)"
    safe_env_decode_value "$value"
    printf '\n'
}