Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

6.9 KB
Raw Blame
#!/usr/bin/env bash
# Regression: Docker full-model hot-swap failures must restore the last
# known-good model config and recreate llama-server from that config. AMD runs
# the same llama.cpp container as NVIDIA: /health, crash-loop detection and
# the rollback recreate apply to both.

set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TARGET="$ROOT_DIR/scripts/bootstrap-upgrade.sh"

fail() {
    echo "[FAIL] $*" >&2
    if [[ -n "${tmp:-}" ]]; then
        [[ -f "$tmp/bootstrap.log" ]] && sed 's/^/[bootstrap] /' "$tmp/bootstrap.log" >&2
        [[ -f "${install_dir:-}/.env" ]] && sed 's/^/[env] /' "$install_dir/.env" >&2
        [[ -f "${install_dir:-}/config/llama-server/models.ini" ]] && sed 's/^/[models.ini] /' "$install_dir/config/llama-server/models.ini" >&2
        [[ -f "${docker_calls:-}" ]] && sed 's/^/[docker] /' "$docker_calls" >&2
    fi
    exit 1
}

pass() {
    echo "[PASS] $*"
}

tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT

fakebin="$tmp/bin"
install_dir="$tmp/install"
docker_calls="$tmp/docker-calls.log"
mkdir -p "$fakebin" "$install_dir/data/models" "$install_dir/config/llama-server" "$install_dir/config/litellm"
# Exercise the production policy gate before both the promotion and rollback.
mkdir -p "$install_dir/scripts"
cp "$ROOT_DIR/scripts/compose-cache-policy.py" "$install_dir/scripts/"

# Keep the Docker-only fixture independent of the operator's managed Pixel.
mkdir -p "$tmp/owner-home"
cat > "$fakebin/getent" <<'EOF'
#!/usr/bin/env bash
[[ "${1:-}" == passwd && -n "${2:-}" ]] || exit 2
printf '%s:x:1000:1000:fixture:%s:/bin/bash\n' "$2" "${ODS_FIXTURE_OWNER_HOME:?}"
EOF
chmod +x "$fakebin/getent"

cat > "$fakebin/curl" <<'EOF'
#!/usr/bin/env bash
case " $* " in
  *" -sI "*)
    printf 'HTTP/2 200\r\ncontent-length: 10\r\n\r\n'
    exit 0
    ;;
esac
printf 'health:%s\n' "$*" >> "${ODS_FAKE_CURL_LOG:?}"
if [[ " $* " == *"11434/health"* ]] && grep -q '^GGUF_FILE=Bootstrap.gguf$' .env 2>/dev/null; then
  printf '{"status":"ok"}\n'
  exit 0
fi
exit 7
EOF
chmod +x "$fakebin/curl"

cat > "$fakebin/sleep" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
chmod +x "$fakebin/sleep"

cat > "$fakebin/uname" <<'EOF'
#!/usr/bin/env bash
printf 'Linux\n'
EOF
chmod +x "$fakebin/uname"

cat > "$fakebin/flock" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
chmod +x "$fakebin/flock"

cat > "$fakebin/docker" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

env_value() {
    local key="$1"
    [[ -f .env ]] || return 0
    grep -E "^${key}=" .env 2>/dev/null | head -1 | cut -d= -f2-
}

case "${1:-}" in
    info)
        exit 0
        ;;
    compose)
        if [[ "${2:-}" == "version" ]]; then
            exit 0
        fi
        if [[ " $* " == *" up -d --force-recreate --no-deps llama-server "* ]]; then
            active_gguf="$(env_value GGUF_FILE)"
            printf 'compose-up:%s\n' "$active_gguf" >> "${ODS_FAKE_DOCKER_LOG:?}"
            exit 0
        fi
        exit 0
        ;;
    ps)
        if [[ " $* " == *"name=ods-llama-server"* ]]; then
            printf 'ods-llama-server\n'
        fi
        exit 0
        ;;
    inspect)
        if [[ "${2:-}" == "ods-llama-server" ]]; then
            if [[ " $* " == *".State.Status"* ]]; then
                printf 'restarting true 1\n'
                exit 0
            fi
            printf '/app/llama-server --model /models/%s --ctx-size %s\n' \
                "$(env_value GGUF_FILE)" "$(env_value CTX_SIZE)"
        fi
        exit 0
        ;;
esac

exit 0
EOF
chmod +x "$fakebin/docker"

run_rollback_case() {
    local backend="$1"
    rm -rf "$install_dir" "$docker_calls" "$tmp/curl-calls.log" "$tmp/bootstrap.log"
    mkdir -p "$install_dir/data/models" "$install_dir/config/llama-server" "$install_dir/scripts"
    cp "$ROOT_DIR/scripts/compose-cache-policy.py" "$install_dir/scripts/"
    cat > "$install_dir/.env" <<EOF
GGUF_FILE=Bootstrap.gguf
LLM_MODEL=bootstrap-model
MAX_CONTEXT=8192
CTX_SIZE=8192
GPU_BACKEND=${backend}
OLLAMA_PORT=11434
EOF

    cat > "$install_dir/config/llama-server/models.ini" <<'EOF'
[bootstrap-model]
filename = Bootstrap.gguf
load-on-startup = true
n-ctx = 8192
EOF

    printf -- '-f docker-compose.base.yml -f docker-compose.%s.yml\n' "$backend" > "$install_dir/.compose-flags"

    printf 'bootstrap\n' > "$install_dir/data/models/Bootstrap.gguf"
    printf 'full-model\n' > "$install_dir/data/models/Full.gguf"

    curl_calls="$tmp/curl-calls.log"

    set +e
    PATH="$fakebin:$PATH" ODS_FIXTURE_OWNER_HOME="$tmp/owner-home" ODS_FAKE_DOCKER_LOG="$docker_calls" ODS_FAKE_CURL_LOG="$curl_calls" bash "$TARGET" \
        "$install_dir" \
        "Full.gguf" \
        "https://example.invalid/Full.gguf" \
        "" \
        "full-model" \
        "32768" \
        "Bootstrap.gguf" \
        > "$tmp/bootstrap.log" 2>&1
    rc=$?
    set -e

    [[ $rc -ne 0 ]] || fail "bootstrap-upgrade must fail when Docker llama-server never becomes healthy"
    grep -q '^GGUF_FILE=Bootstrap.gguf$' "$install_dir/.env" \
        || fail "Docker hot-swap failure must restore previous GGUF_FILE"
    grep -q '^LLM_MODEL=bootstrap-model$' "$install_dir/.env" \
        || fail "Docker hot-swap failure must restore previous LLM_MODEL"
    grep -q '^CTX_SIZE=8192$' "$install_dir/.env" \
        || fail "Docker hot-swap failure must restore previous CTX_SIZE"
    grep -q 'filename = Bootstrap.gguf' "$install_dir/config/llama-server/models.ini" \
        || fail "Docker hot-swap failure must restore previous models.ini"
    if grep -q '/api/v1/' "$curl_calls"; then
        fail "the $backend hot-swap must probe llama.cpp /health, not a Lemonade /api/v1 route"
    fi
    grep -q 'compose-up:Full.gguf' "$docker_calls" \
        || fail "test did not exercise the full-model compose recreate"
    grep -q 'compose-up:Bootstrap.gguf' "$docker_calls" \
        || fail "rollback must recreate llama-server from the restored bootstrap config"
    grep -q 'Restoring previous active model config after Docker llama-server swap failure' "$tmp/bootstrap.log" \
        || fail "bootstrap-upgrade should log the Docker rollback"
    grep -q 'llama-server container exited or is restarting while loading the full model' "$tmp/bootstrap.log" \
        || fail "bootstrap-upgrade should detect a failed llama-server container before waiting for the full health timeout"
    grep -q 'continuing within restart grace' "$tmp/bootstrap.log" \
        || fail "bootstrap-upgrade should tolerate transient llama-server restarts before rollback"
    health_attempts=$(grep -c '^health:' "$curl_calls" 2>/dev/null || true)
    [[ "$health_attempts" -lt 60 ]] \
        || fail "failed Docker hot-swap should not wait for all health attempts after the container is restarting"
    grep -q '"status": "failed"' "$install_dir/data/bootstrap-status.json" \
        || fail "failed Docker hot-swap must mark bootstrap-status failed"

    pass "$backend Docker hot-swap failure restores previous active model config"
}

run_rollback_case nvidia
run_rollback_case amd