~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Behaviour of the stop/restart probes in compose-integration-test.yml
(Compose) and docker-tests.yml (docker run).
The step's ``run:`` script is executed with a stub ``docker`` on PATH under
the shell command GitHub Actions derives from the step's ``shell:`` key
(``bash -e`` when the key is absent, ``bash --noprofile --norc -eo pipefail``
for ``shell: bash``), so these tests check which stop grace the probe really
applies and that it fails closed in the shell CI actually runs.
"""
# allow: no-sut-import — the code under test is a GitHub Actions workflow step
from __future__ import annotations
import json
import os
import shutil
import subprocess
from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "compose-integration-test.yml"
STEP_NAME = "Verify authenticated persistence across Compose restart"
pytestmark = pytest.mark.skipif(
shutil.which("bash") is None or shutil.which("jq") is None,
reason="needs bash and jq, as on the GitHub runner",
)
STUB_DOCKER = r"""#!/usr/bin/env bash
printf '%s\n' "$*" >> "$STUB_LOG"
case "$*" in
"compose config --format json")
[ "${STUB_CONFIG_FAIL:-}" = 1 ] && exit 1
cat "$STUB_CONFIG_JSON" ;;
"compose ps -q local-deep-research") echo stub-cid ;;
*"{{.State.ExitCode}}"*) echo "${STUB_EXIT_CODE:-0}" ;;
*"{{.State.OOMKilled}}"*) echo "${STUB_OOM_KILLED:-false}" ;;
*"{{.State.Error}}"*)
[ "${STUB_STATE_ERROR_FAIL:-}" = 1 ] && exit 1
echo "${STUB_STATE_ERROR_TEXT:-}" ;;
esac
exit 0
"""
# How GitHub Actions invokes a `run:` script for each `shell:` value
# (https://docs.github.com/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepsshell).
# A missing key on a Linux runner means `bash -e {0}`: no pipefail.
GITHUB_SHELLS = {
None: ["bash", "-e"],
"bash": ["bash", "--noprofile", "--norc", "-eo", "pipefail"],
"sh": ["sh", "-e"],
}
def _step() -> tuple[list[str], str]:
"""Return the shell argv GitHub would use for the step, and its script."""
workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
job = workflow["jobs"]["compose-up"]
matches = [s for s in job["steps"] if s.get("name") == STEP_NAME]
assert len(matches) == 1, f"expected one {STEP_NAME!r} step"
step = matches[0]
job_default = job.get("defaults", {}).get("run", {}).get("shell")
wf_default = workflow.get("defaults", {}).get("run", {}).get("shell")
shell = step.get("shell") or job_default or wf_default
assert shell in GITHUB_SHELLS, f"unmodelled shell: {shell!r}"
return GITHUB_SHELLS[shell], step["run"]
def _run(tmp_path: Path, service: dict, **env) -> tuple[int, str, list[str]]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
docker = bin_dir / "docker"
docker.write_text(STUB_DOCKER, encoding="utf-8")
docker.chmod(0o755)
config = tmp_path / "config.json"
config.write_text(
json.dumps({"services": {"local-deep-research": service}}),
encoding="utf-8",
)
log = tmp_path / "docker.log"
log.touch()
shell_argv, step_script = _step()
script = tmp_path / "step.sh"
script.write_text(step_script, encoding="utf-8")
proc = subprocess.run(
[*shell_argv, str(script)],
capture_output=True,
text=True,
timeout=60,
env={
**os.environ,
"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
"STUB_LOG": str(log),
"STUB_CONFIG_JSON": str(config),
**env,
},
)
calls = log.read_text(encoding="utf-8").splitlines()
return proc.returncode, proc.stdout + proc.stderr, calls
def _stop_calls(calls: list[str]) -> list[str]:
return [c for c in calls if c.startswith("compose stop")]
def test_declared_stop_grace_period_is_what_the_probe_applies(tmp_path):
rc, output, calls = _run(tmp_path, {"stop_grace_period": "1m0s"})
assert rc == 0, output
stops = _stop_calls(calls)
assert len(stops) == 2, calls
assert all(c == "compose stop local-deep-research" for c in stops), stops
assert "::warning::" not in output
def test_missing_stop_grace_period_is_overridden_loudly(tmp_path):
rc, output, calls = _run(tmp_path, {})
assert rc == 0, output
stops = _stop_calls(calls)
assert len(stops) == 2, calls
assert all(
c == "compose stop --timeout 60 local-deep-research" for c in stops
), stops
assert output.count("::warning::") == 2
assert "stop_grace_period" in output
assert "#6918" in output
@pytest.mark.parametrize("exit_code", ["137", "1"])
def test_forced_or_failed_stop_fails_the_step(tmp_path, exit_code):
rc, output, _ = _run(
tmp_path, {"stop_grace_period": "1m0s"}, STUB_EXIT_CODE=exit_code
)
assert rc != 0
assert f"Unexpected Compose shutdown exit: {exit_code}" in output
def test_unreadable_compose_config_fails_closed(tmp_path):
rc, _, calls = _run(tmp_path, {}, STUB_CONFIG_FAIL="1")
assert rc != 0
assert not _stop_calls(calls)
def test_failed_state_error_inspect_fails_closed(tmp_path):
"""A failing ``docker inspect`` inside ``test -z "$(...)"`` is ignored
by ``-e``; the step must not treat it as an empty State.Error."""
rc, _, calls = _run(
tmp_path, {"stop_grace_period": "1m0s"}, STUB_STATE_ERROR_FAIL="1"
)
assert rc != 0
assert len(_stop_calls(calls)) == 1, calls
def _run_native_stop_step(tmp_path: Path, **env) -> tuple[int, str, list[str]]:
"""Run docker-tests.yml's docker-run stop/restart step with a stub docker."""
workflow = yaml.safe_load(
(REPO_ROOT / ".github" / "workflows" / "docker-tests.yml").read_text(
encoding="utf-8"
)
)
steps = workflow["jobs"]["ldr-production-smoke-test"]["steps"]
matches = [
s
for s in steps
if s.get("name") == "Verify persistence and bounded stop/restart"
]
assert len(matches) == 1
assert matches[0].get("shell") == "bash"
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
docker = bin_dir / "docker"
docker.write_text(STUB_DOCKER, encoding="utf-8")
docker.chmod(0o755)
log = tmp_path / "docker.log"
log.touch()
script = tmp_path / "step.sh"
script.write_text(matches[0]["run"], encoding="utf-8")
proc = subprocess.run(
[*GITHUB_SHELLS["bash"], str(script)],
capture_output=True,
text=True,
timeout=60,
env={
**os.environ,
"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
"STUB_LOG": str(log),
**env,
},
)
calls = log.read_text(encoding="utf-8").splitlines()
return proc.returncode, proc.stdout + proc.stderr, calls
def _native_stops(calls: list[str]) -> list[str]:
return [c for c in calls if c.startswith("stop")]
def test_docker_run_smoke_warns_that_it_overrides_the_default_stop_grace(
tmp_path,
):
"""The docker-run smoke stops with --timeout 60 while README users get
Docker's 10s default (#6918); it must say so like the Compose step."""
rc, output, calls = _run_native_stop_step(tmp_path)
assert rc == 0, output
assert _native_stops(calls) == ["stop --timeout 60 ldr-prod-test"] * 2, (
calls
)
assert output.count("::warning::") == 2
assert "#6918" in output
@pytest.mark.parametrize("exit_code", ["0", "143"])
def test_docker_run_smoke_accepts_graceful_exit_codes(tmp_path, exit_code):
rc, output, calls = _run_native_stop_step(
tmp_path, STUB_EXIT_CODE=exit_code
)
assert rc == 0, output
assert len(_native_stops(calls)) == 2, calls
@pytest.mark.parametrize("exit_code", ["137", "1"])
def test_docker_run_smoke_forced_or_failed_stop_fails_the_step(
tmp_path, exit_code
):
rc, output, calls = _run_native_stop_step(
tmp_path, STUB_EXIT_CODE=exit_code
)
assert rc != 0
assert f"Unexpected shutdown exit: {exit_code}" in output
assert len(_native_stops(calls)) == 1, calls
def test_docker_run_smoke_oom_kill_fails_the_step(tmp_path):
rc, _, calls = _run_native_stop_step(tmp_path, STUB_OOM_KILLED="true")
assert rc != 0
assert len(_native_stops(calls)) == 1, calls
def test_docker_run_smoke_failed_state_error_inspect_fails_closed(tmp_path):
"""A failing ``docker inspect`` inside ``test -z "$(...)"`` is ignored
by ``-e``; the step must not treat it as an empty State.Error."""
rc, _, calls = _run_native_stop_step(tmp_path, STUB_STATE_ERROR_FAIL="1")
assert rc != 0
assert len(_native_stops(calls)) == 1, calls
def test_docker_run_smoke_non_empty_state_error_fails_the_step(tmp_path):
rc, _, calls = _run_native_stop_step(tmp_path, STUB_STATE_ERROR_TEXT="boom")
assert rc != 0
assert len(_native_stops(calls)) == 1, calls