Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

8.7 KB
Raw Blame
"""Behaviour of the stop/restart probes in compose-integration-test.yml
(Compose) and docker-tests.yml (docker run).

The step's ``run:`` script is executed with a stub ``docker`` on PATH under
the shell command GitHub Actions derives from the step's ``shell:`` key
(``bash -e`` when the key is absent, ``bash --noprofile --norc -eo pipefail``
for ``shell: bash``), so these tests check which stop grace the probe really
applies and that it fails closed in the shell CI actually runs.
"""

# allow: no-sut-import — the code under test is a GitHub Actions workflow step

from __future__ import annotations

import json
import os
import shutil
import subprocess
from pathlib import Path

import pytest
import yaml

REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "compose-integration-test.yml"
STEP_NAME = "Verify authenticated persistence across Compose restart"

pytestmark = pytest.mark.skipif(
    shutil.which("bash") is None or shutil.which("jq") is None,
    reason="needs bash and jq, as on the GitHub runner",
)

STUB_DOCKER = r"""#!/usr/bin/env bash
printf '%s\n' "$*" >> "$STUB_LOG"
case "$*" in
  "compose config --format json")
    [ "${STUB_CONFIG_FAIL:-}" = 1 ] && exit 1
    cat "$STUB_CONFIG_JSON" ;;
  "compose ps -q local-deep-research") echo stub-cid ;;
  *"{{.State.ExitCode}}"*) echo "${STUB_EXIT_CODE:-0}" ;;
  *"{{.State.OOMKilled}}"*) echo "${STUB_OOM_KILLED:-false}" ;;
  *"{{.State.Error}}"*)
    [ "${STUB_STATE_ERROR_FAIL:-}" = 1 ] && exit 1
    echo "${STUB_STATE_ERROR_TEXT:-}" ;;
esac
exit 0
"""

# How GitHub Actions invokes a `run:` script for each `shell:` value
# (https://docs.github.com/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepsshell).
# A missing key on a Linux runner means `bash -e {0}`: no pipefail.
GITHUB_SHELLS = {
    None: ["bash", "-e"],
    "bash": ["bash", "--noprofile", "--norc", "-eo", "pipefail"],
    "sh": ["sh", "-e"],
}


def _step() -> tuple[list[str], str]:
    """Return the shell argv GitHub would use for the step, and its script."""
    workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
    job = workflow["jobs"]["compose-up"]
    matches = [s for s in job["steps"] if s.get("name") == STEP_NAME]
    assert len(matches) == 1, f"expected one {STEP_NAME!r} step"
    step = matches[0]
    job_default = job.get("defaults", {}).get("run", {}).get("shell")
    wf_default = workflow.get("defaults", {}).get("run", {}).get("shell")
    shell = step.get("shell") or job_default or wf_default
    assert shell in GITHUB_SHELLS, f"unmodelled shell: {shell!r}"
    return GITHUB_SHELLS[shell], step["run"]


def _run(tmp_path: Path, service: dict, **env) -> tuple[int, str, list[str]]:
    bin_dir = tmp_path / "bin"
    bin_dir.mkdir()
    docker = bin_dir / "docker"
    docker.write_text(STUB_DOCKER, encoding="utf-8")
    docker.chmod(0o755)
    config = tmp_path / "config.json"
    config.write_text(
        json.dumps({"services": {"local-deep-research": service}}),
        encoding="utf-8",
    )
    log = tmp_path / "docker.log"
    log.touch()
    shell_argv, step_script = _step()
    script = tmp_path / "step.sh"
    script.write_text(step_script, encoding="utf-8")
    proc = subprocess.run(
        [*shell_argv, str(script)],
        capture_output=True,
        text=True,
        timeout=60,
        env={
            **os.environ,
            "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
            "STUB_LOG": str(log),
            "STUB_CONFIG_JSON": str(config),
            **env,
        },
    )
    calls = log.read_text(encoding="utf-8").splitlines()
    return proc.returncode, proc.stdout + proc.stderr, calls


def _stop_calls(calls: list[str]) -> list[str]:
    return [c for c in calls if c.startswith("compose stop")]


def test_declared_stop_grace_period_is_what_the_probe_applies(tmp_path):
    rc, output, calls = _run(tmp_path, {"stop_grace_period": "1m0s"})
    assert rc == 0, output
    stops = _stop_calls(calls)
    assert len(stops) == 2, calls
    assert all(c == "compose stop local-deep-research" for c in stops), stops
    assert "::warning::" not in output


def test_missing_stop_grace_period_is_overridden_loudly(tmp_path):
    rc, output, calls = _run(tmp_path, {})
    assert rc == 0, output
    stops = _stop_calls(calls)
    assert len(stops) == 2, calls
    assert all(
        c == "compose stop --timeout 60 local-deep-research" for c in stops
    ), stops
    assert output.count("::warning::") == 2
    assert "stop_grace_period" in output
    assert "#6918" in output


@pytest.mark.parametrize("exit_code", ["137", "1"])
def test_forced_or_failed_stop_fails_the_step(tmp_path, exit_code):
    rc, output, _ = _run(
        tmp_path, {"stop_grace_period": "1m0s"}, STUB_EXIT_CODE=exit_code
    )
    assert rc != 0
    assert f"Unexpected Compose shutdown exit: {exit_code}" in output


def test_unreadable_compose_config_fails_closed(tmp_path):
    rc, _, calls = _run(tmp_path, {}, STUB_CONFIG_FAIL="1")
    assert rc != 0
    assert not _stop_calls(calls)


def test_failed_state_error_inspect_fails_closed(tmp_path):
    """A failing ``docker inspect`` inside ``test -z "$(...)"`` is ignored
    by ``-e``; the step must not treat it as an empty State.Error."""
    rc, _, calls = _run(
        tmp_path, {"stop_grace_period": "1m0s"}, STUB_STATE_ERROR_FAIL="1"
    )
    assert rc != 0
    assert len(_stop_calls(calls)) == 1, calls


def _run_native_stop_step(tmp_path: Path, **env) -> tuple[int, str, list[str]]:
    """Run docker-tests.yml's docker-run stop/restart step with a stub docker."""
    workflow = yaml.safe_load(
        (REPO_ROOT / ".github" / "workflows" / "docker-tests.yml").read_text(
            encoding="utf-8"
        )
    )
    steps = workflow["jobs"]["ldr-production-smoke-test"]["steps"]
    matches = [
        s
        for s in steps
        if s.get("name") == "Verify persistence and bounded stop/restart"
    ]
    assert len(matches) == 1
    assert matches[0].get("shell") == "bash"
    bin_dir = tmp_path / "bin"
    bin_dir.mkdir()
    docker = bin_dir / "docker"
    docker.write_text(STUB_DOCKER, encoding="utf-8")
    docker.chmod(0o755)
    log = tmp_path / "docker.log"
    log.touch()
    script = tmp_path / "step.sh"
    script.write_text(matches[0]["run"], encoding="utf-8")
    proc = subprocess.run(
        [*GITHUB_SHELLS["bash"], str(script)],
        capture_output=True,
        text=True,
        timeout=60,
        env={
            **os.environ,
            "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
            "STUB_LOG": str(log),
            **env,
        },
    )
    calls = log.read_text(encoding="utf-8").splitlines()
    return proc.returncode, proc.stdout + proc.stderr, calls


def _native_stops(calls: list[str]) -> list[str]:
    return [c for c in calls if c.startswith("stop")]


def test_docker_run_smoke_warns_that_it_overrides_the_default_stop_grace(
    tmp_path,
):
    """The docker-run smoke stops with --timeout 60 while README users get
    Docker's 10s default (#6918); it must say so like the Compose step."""
    rc, output, calls = _run_native_stop_step(tmp_path)
    assert rc == 0, output
    assert _native_stops(calls) == ["stop --timeout 60 ldr-prod-test"] * 2, (
        calls
    )
    assert output.count("::warning::") == 2
    assert "#6918" in output


@pytest.mark.parametrize("exit_code", ["0", "143"])
def test_docker_run_smoke_accepts_graceful_exit_codes(tmp_path, exit_code):
    rc, output, calls = _run_native_stop_step(
        tmp_path, STUB_EXIT_CODE=exit_code
    )
    assert rc == 0, output
    assert len(_native_stops(calls)) == 2, calls


@pytest.mark.parametrize("exit_code", ["137", "1"])
def test_docker_run_smoke_forced_or_failed_stop_fails_the_step(
    tmp_path, exit_code
):
    rc, output, calls = _run_native_stop_step(
        tmp_path, STUB_EXIT_CODE=exit_code
    )
    assert rc != 0
    assert f"Unexpected shutdown exit: {exit_code}" in output
    assert len(_native_stops(calls)) == 1, calls


def test_docker_run_smoke_oom_kill_fails_the_step(tmp_path):
    rc, _, calls = _run_native_stop_step(tmp_path, STUB_OOM_KILLED="true")
    assert rc != 0
    assert len(_native_stops(calls)) == 1, calls


def test_docker_run_smoke_failed_state_error_inspect_fails_closed(tmp_path):
    """A failing ``docker inspect`` inside ``test -z "$(...)"`` is ignored
    by ``-e``; the step must not treat it as an empty State.Error."""
    rc, _, calls = _run_native_stop_step(tmp_path, STUB_STATE_ERROR_FAIL="1")
    assert rc != 0
    assert len(_native_stops(calls)) == 1, calls


def test_docker_run_smoke_non_empty_state_error_fails_the_step(tmp_path):
    rc, _, calls = _run_native_stop_step(tmp_path, STUB_STATE_ERROR_TEXT="boom")
    assert rc != 0
    assert len(_native_stops(calls)) == 1, calls