Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

ci: qualify native images and Compose lifecycle (#6936)

LearningCircuit68eeff608f55
diff --git a/.github/workflows/compose-integration-test.yml b/.github/workflows/compose-integration-test.yml
index a15f331ff0..bc80f4d6af 100644
--- a/.github/workflows/compose-integration-test.yml
+++ b/.github/workflows/compose-integration-test.yml
@@ -19,14 +19,31 @@ name: Compose Integration Test
# published image, not main's build override. PR #3962 tried adding this
# to the daily cron and was closed for that reason.
#
-# Tests the docker-compose.yml as users actually run it — no test-only
-# overrides. The model pre-pull was removed from the compose itself in this
-# same PR so the stack starts in seconds rather than waiting on a multi-GB
-# download that the test doesn't need.
+# Tests the bundled docker-compose.yml without service overrides, including
+# authenticated persistence across stop/restart. The restart probe's stop
+# honours the service's stop_grace_period; only while the bundled file
+# declares none (Docker's 10s default, #6918) does it pass an explicit
+# --timeout 60 and emit a warning (see that step). Model pulling remains
+# separate from startup: the opt-in `run-model-probe` input pulls a small model
+# and checks real inference. It defaults to false and release.yml's
+# compose-integration-gate does not set it, so releases do not run the model
+# probe; use a manual dispatch with the input enabled for that.
on:
workflow_call:
+ inputs:
+ run-model-probe:
+ description: Pull a small model and verify inference from the LDR container
+ required: false
+ type: boolean
+ default: false
workflow_dispatch:
+ inputs:
+ run-model-probe:
+ description: Pull a small model and verify inference from the LDR container
+ required: false
+ type: boolean
+ default: false
permissions:
contents: read
@@ -60,20 +77,22 @@ jobs:
with:
context: .
# Same target as docker-tests.yml's `ldr-prod` build, so its cache
- # layers (scope=ldr-prod, populated by ci-gate during the same
- # release pipeline) are reusable here. Explicit target guards
+ # layers (scope=ldr-prod-amd64, written by that workflow's amd64
+ # smoke job on non-pull_request runs) are reusable here. release.yml
+ # runs ci-gate in parallel with this gate, so a release run can only
+ # reuse layers from earlier runs, not from its own. Explicit target guards
# against future Dockerfile reorderings making the default stage
# something other than `ldr`.
target: ldr
load: true
tags: localdeepresearch/local-deep-research:latest
- # Read from both our own scope and the shared `ldr-prod` scope —
+ # Read from both our own scope and the shared `ldr-prod-amd64` scope —
# whichever has the layers wins. Falls back to a fresh build if
# neither does (e.g. on a brand-new branch). Only write to our
# own scope so we don't poison the cross-workflow cache.
cache-from: |
type=gha,scope=compose-integration
- type=gha,scope=ldr-prod
+ type=gha,scope=ldr-prod-amd64
cache-to: type=gha,mode=max,scope=compose-integration
- name: Bring up the stack
@@ -87,7 +106,7 @@ jobs:
run: docker compose up -d --no-build
- name: Wait for the stack to be healthy and serving
- # Budget: 6 min covers cold daemon startup + LDR migrations + flask
+ # Budget: 6 min covers cold daemon startup + LDR migrations + application
# boot + slow CI runners. Without the model pull this completes in
# ~1-2 min on a warm runner.
#
@@ -177,6 +196,63 @@ jobs:
echo "::error::LDR HTTP probe failed after ~150s"
exit 1
+ - name: Pull and exercise a local model
+ if: inputs.run-model-probe == true
+ timeout-minutes: 10
+ run: |
+ docker compose exec -T ollama ollama pull qwen2.5:0.5b
+ compose_ldr_id=$(docker compose ps -q local-deep-research)
+ docker cp tests/ci/standalone/compose_ollama.py "$compose_ldr_id":/tmp/compose_ollama.py
+ docker compose exec -T local-deep-research /install/.venv/bin/python /tmp/compose_ollama.py
+
+ - name: Verify authenticated persistence across Compose restart
+ # The stop must exercise the grace period users actually get. When the
+ # bundled compose declares stop_grace_period, a plain `compose stop`
+ # applies it. While it declares none, users get Docker's 10s default,
+ # which equals uvicorn's own drain budget and can SIGKILL the lifespan
+ # cleanup (#6918; pinned by the strict-xfail
+ # test_stop_grace_period_covers_the_shutdown_budget). This probe then
+ # passes --timeout 60 so it checks that shutdown completes at all, and
+ # says loudly that it is not testing the shipped grace.
+ #
+ # Fail closed: `shell: bash` makes GitHub run this with
+ # `bash --noprofile --norc -eo pipefail` instead of the default
+ # `bash -e`, and the config is captured in its own assignment
+ # before jq reads it, so a failed `docker compose config` aborts
+ # the step instead of looking like an undeclared grace. Every
+ # docker inspect result is assigned before it is tested, because
+ # -e ignores a failing $(...) used as a command argument.
+ shell: bash
+ run: |
+ compose_config=$(docker compose config --format json)
+ compose_stop_grace=$(jq -r '.services["local-deep-research"].stop_grace_period // empty' <<<"$compose_config")
+ stop_without_kill() {
+ local compose_ldr_id compose_exit_code compose_oom_killed compose_state_error
+ compose_ldr_id=$(docker compose ps -q local-deep-research)
+ if [ -n "$compose_stop_grace" ]; then
+ docker compose stop local-deep-research
+ else
+ echo "::warning::docker-compose.yml sets no stop_grace_period for local-deep-research, so users get Docker's 10s default (#6918). This probe stops with --timeout 60 and does not test the shipped grace period."
+ docker compose stop --timeout 60 local-deep-research
+ fi
+ compose_exit_code=$(docker inspect "$compose_ldr_id" --format '{{.State.ExitCode}}')
+ case "$compose_exit_code" in
+ 0|143) ;;
+ *) echo "Unexpected Compose shutdown exit: $compose_exit_code"; return 1 ;;
+ esac
+ compose_oom_killed=$(docker inspect "$compose_ldr_id" --format '{{.State.OOMKilled}}')
+ test "$compose_oom_killed" = false
+ compose_state_error=$(docker inspect "$compose_ldr_id" --format '{{.State.Error}}')
+ test -z "$compose_state_error"
+ }
+ compose_ldr_id=$(docker compose ps -q local-deep-research)
+ docker cp tests/ci/standalone/production_lifecycle.py "$compose_ldr_id":/tmp/production_lifecycle.py
+ docker compose exec -T local-deep-research /install/.venv/bin/python /tmp/production_lifecycle.py seed /data/lifecycle-state.json
+ stop_without_kill
+ docker compose start local-deep-research
+ docker compose exec -T local-deep-research /install/.venv/bin/python /tmp/production_lifecycle.py verify /data/lifecycle-state.json
+ stop_without_kill
+
- name: Dump compose state and logs
if: always()
run: |
diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml
index 384c8e9fd4..aaec752f70 100644
--- a/.github/workflows/docker-tests.yml
+++ b/.github/workflows/docker-tests.yml
@@ -115,16 +115,10 @@ jobs:
- 'package.json'
- 'package-lock.json'
- 'vite.config.js'
- # Web-layer sources gate the production smoke test because it is
- # the ONLY job that actually boots the app. Nothing else does:
- # no test uses `with TestClient(app)`, so `lifespan()` — the
- # from-scratch rewrite of Flask's app_factory startup — is never
- # executed by the merge gate. Before this line, a PR touching
- # only web/fastapi_app.py matched no filter at all and was
- # verified by zero jobs that could detect it failing to start.
- # The smoke test builds the production image, runs it, and polls
- # health, so it catches exactly that class.
- - 'src/local_deep_research/web/**'
+ # Boot the production image on both native architectures and
+ # verify encrypted data access across a clean stop/restart.
+ - 'src/local_deep_research/**'
+ - 'tests/ci/standalone/production_lifecycle.py'
- '.github/workflows/docker-tests.yml'
accessibility:
# Theme palettes (and the loader/schema that validate them) are
@@ -1554,11 +1548,19 @@ jobs:
# Smoke test for the production ldr image
ldr-production-smoke-test:
- runs-on: ubuntu-latest
- name: Production Image Smoke Test
+ runs-on: ${{ matrix.runner }}
+ name: Production Image Smoke Test (${{ matrix.arch }})
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - arch: amd64
+ runner: ubuntu-latest
+ - arch: arm64
+ runner: ubuntu-24.04-arm
needs: detect-changes
if: needs.detect-changes.outputs.docker == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true
- timeout-minutes: 30
+ timeout-minutes: 40
permissions:
contents: read
@@ -1581,10 +1583,16 @@ jobs:
context: .
target: ldr
load: true
+ platforms: linux/${{ matrix.arch }}
tags: ldr-prod
# Cache poisoning protection: only read/write cache on trusted events (not PRs from forks)
- cache-from: ${{ github.event_name != 'pull_request' && 'type=gha,scope=ldr-prod' || '' }}
- cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=ldr-prod' || '' }}
+ cache-from: ${{ github.event_name != 'pull_request' && format('type=gha,scope=ldr-prod-{0}', matrix.arch) || '' }}
+ cache-to: ${{ github.event_name != 'pull_request' && format('type=gha,mode=max,scope=ldr-prod-{0}', matrix.arch) || '' }}
+
+ - name: Verify native image architecture
+ env:
+ EXPECTED_ARCH: ${{ matrix.arch }}
+ run: test "$(docker image inspect ldr-prod --format '{{.Architecture}}')" = "$EXPECTED_ARCH"
- name: Verify file ownership and playwright access
run: |
@@ -1648,10 +1656,19 @@ jobs:
# Run with the same security settings as docker-compose.yml so CI
# catches capability regressions (e.g., missing cap_add that breaks
# the entrypoint's chown/setpriv in restricted environments).
+ # --stop-timeout 60 deliberately differs from the README `docker run`
+ # recipe, which sets none and so gets Docker's 10s default (the
+ # Compose equivalent is #6918). The stop step below always passes
+ # an explicit `docker stop --timeout 60`, which overrides this
+ # flag, so the flag only matters for stops this job does not issue
+ # itself; the stop step says loudly that it does not test the
+ # shipped grace period.
docker run -d \
--name ldr-prod-test \
+ --stop-timeout 60 \
-p 5001:5000 \
-v "$PWD/ci-data-prod":/data \
+ -v "$PWD/tests/ci/standalone/production_lifecycle.py":/tmp/production_lifecycle.py:ro \
--security-opt no-new-privileges:true \
--cap-drop ALL \
--cap-add CHOWN \
@@ -1661,7 +1678,9 @@ jobs:
--cap-add SETGID \
-e LDR_DATA_DIR=/data \
-e LDR_DISABLE_RATE_LIMITING=true \
- -e SECRET_KEY=test-secret-key-for-ci \
+ -e LDR_APP_ALLOW_REGISTRATIONS=true \
+ -e LDR_BOOTSTRAP_ALLOW_UNENCRYPTED=false \
+ -e LDR_WEB_QUEUE_PROCESSOR_ENABLED=true \
ldr-prod
- name: Wait for healthy status
@@ -1684,11 +1703,42 @@ jobs:
docker logs ldr-prod-test
exit 1
- - name: Stop container
+ - name: Verify persistence and bounded stop/restart
+ # `shell: bash` adds pipefail to GitHub's default `bash -e`; every
+ # docker inspect result is assigned before it is tested, because -e
+ # ignores a failing $(...) used as a command argument.
+ shell: bash
+ run: |
+ stop_without_kill() {
+ local lifecycle_exit_code lifecycle_oom_killed lifecycle_state_error
+ echo "::warning::The README docker run recipe sets no stop timeout, so users get Docker's 10s default (the Compose equivalent is #6918, which does not change the README recipe). This probe stops with --timeout 60 and does not test the shipped grace period."
+ docker stop --timeout 60 ldr-prod-test
+ lifecycle_exit_code=$(docker inspect ldr-prod-test --format '{{.State.ExitCode}}')
+ # A graceful shutdown exits 0, or 143 if the SIGTERM status
+ # propagates. A forced SIGKILL exits 137 and must fail.
+ case "$lifecycle_exit_code" in
+ 0|143) ;;
+ *) echo "Unexpected shutdown exit: $lifecycle_exit_code"; return 1 ;;
+ esac
+ lifecycle_oom_killed=$(docker inspect ldr-prod-test --format '{{.State.OOMKilled}}')
+ test "$lifecycle_oom_killed" = false
+ lifecycle_state_error=$(docker inspect ldr-prod-test --format '{{.State.Error}}')
+ test -z "$lifecycle_state_error"
+ }
+ docker exec ldr-prod-test /install/.venv/bin/python /tmp/production_lifecycle.py seed /data/lifecycle-state.json
+ stop_without_kill
+ docker start ldr-prod-test
+ docker exec ldr-prod-test /install/.venv/bin/python /tmp/production_lifecycle.py verify /data/lifecycle-state.json
+ stop_without_kill
+
+ - name: Collect production logs
+ if: always()
+ run: docker logs ldr-prod-test || true
+
+ - name: Remove test container
if: always()
run: |
- docker stop ldr-prod-test || true
- docker rm ldr-prod-test || true
+ docker rm -f ldr-prod-test || true
# Run infrastructure tests (conditional on path changes)
infrastructure-tests:
diff --git a/tests/ci/standalone/compose_ollama.py b/tests/ci/standalone/compose_ollama.py
new file mode 100644
index 0000000000..c3d8275c25
--- /dev/null
+++ b/tests/ci/standalone/compose_ollama.py
@@ -0,0 +1,50 @@
+"""Verify real Ollama inference over the bundled Compose service network.
+
+Run inside the LDR container after explicitly pulling qwen2.5:0.5b in Ollama.
+This checks container DNS, the configured endpoint, model availability and
+inference transport. It is not a research-quality benchmark.
+"""
+
+import os
+
+import requests
+
+
+MODEL = "qwen2.5:0.5b"
+
+
+def main():
+ endpoint = os.environ["LDR_LLM_OLLAMA_URL"].rstrip("/")
+ with requests.Session() as client:
+ client.trust_env = False
+ response = client.get(endpoint + "/api/tags", timeout=10)
+ response.raise_for_status()
+ model = next(
+ item for item in response.json()["models"] if item["name"] == MODEL
+ )
+ assert model["digest"], "Pulled model has no manifest digest"
+ response = client.post(
+ endpoint + "/api/generate",
+ json={
+ "model": MODEL,
+ "prompt": "Reply with the word ready.",
+ "stream": False,
+ "keep_alive": 0,
+ "options": {
+ "num_predict": 8,
+ "num_ctx": 1024,
+ "temperature": 0,
+ },
+ },
+ timeout=300,
+ )
+ response.raise_for_status()
+ result = response.json()
+ assert result["done"] is True, "Ollama did not complete inference"
+ assert result["response"].strip(), "Ollama returned no generated text"
+ assert result["eval_count"] > 0, "Ollama generated no tokens"
+ print(f"Compose inference passed: {MODEL}, digest={model['digest']}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/ci/standalone/production_lifecycle.py b/tests/ci/standalone/production_lifecycle.py
new file mode 100644
index 0000000000..f6eaa16526
--- /dev/null
+++ b/tests/ci/standalone/production_lifecycle.py
@@ -0,0 +1,134 @@
+"""Exercise the running production image before and after a container restart.
+
+This standalone CLI uses loopback HTTP and a disposable test account. Its state
+file holds generated test credentials and stays in the temporary CI volume.
+"""
+
+import argparse
+import json
+import secrets
+import time
+from contextlib import closing
+from pathlib import Path
+
+import requests
+
+
+def wait_until_ready(base_url):
+ deadline = time.monotonic() + 120
+ with requests.Session() as client:
+ client.trust_env = False
+ while time.monotonic() < deadline:
+ try:
+ response = client.get(base_url + "/api/v1/health", timeout=2)
+ if response.status_code == 200:
+ assert response.json()["status"] == "ok"
+ return
+ except requests.RequestException:
+ pass
+ # CLI readiness polling; this probe is not collected by pytest.
+ time.sleep(1) # allow: unmarked-sleep
+ raise RuntimeError(
+ "Production server did not become ready within 120 seconds"
+ )
+
+
+def csrf(client, base_url):
+ response = client.get(base_url + "/auth/csrf-token", timeout=10)
+ response.raise_for_status()
+ return response.json()["csrf_token"]
+
+
+def login(client, base_url, state):
+ response = client.post(
+ base_url + "/auth/login",
+ data={
+ "username": state["username"],
+ "password": state["password"],
+ "csrf_token": csrf(client, base_url),
+ },
+ allow_redirects=False,
+ timeout=30,
+ )
+ assert response.status_code == 302, "Login failed"
+ response = client.get(base_url + "/auth/check", timeout=10)
+ response.raise_for_status()
+ assert response.json()["username"] == state["username"]
+
+
+def check_saved_setting(client, base_url, value):
+ response = client.get(
+ base_url + "/settings/api/search.iterations", timeout=10
+ )
+ response.raise_for_status()
+ assert response.json()["value"] == value, "Saved setting did not survive"
+ response = client.get(base_url + "/api/v1/health", timeout=10)
+ response.raise_for_status()
+ assert response.json()["subsystems"]["queue_processor"] == "ok"
+ assert response.json()["subsystems"]["db_manager"] == "ok"
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("phase", choices=("seed", "verify"))
+ parser.add_argument("state", type=Path)
+ parser.add_argument("--base-url", default="http://127.0.0.1:5000")
+ args = parser.parse_args()
+ wait_until_ready(args.base_url)
+
+ # Import the actual production SQLCipher wheel on each native architecture.
+ from sqlcipher3 import dbapi2
+
+ with closing(dbapi2.connect(":memory:")) as connection:
+ assert connection.execute("PRAGMA cipher_version").fetchone()[0]
+
+ with requests.Session() as client:
+ client.trust_env = False
+ client.headers["Accept"] = "application/json"
+ if args.phase == "seed":
+ state = {
+ "username": "prod_smoke_" + secrets.token_hex(6),
+ "password": secrets.token_urlsafe(24) + "aA1!",
+ "value": 7,
+ }
+ response = client.post(
+ args.base_url + "/auth/register",
+ data={
+ "username": state["username"],
+ "password": state["password"],
+ "confirm_password": state["password"],
+ "acknowledge": "true",
+ "csrf_token": csrf(client, args.base_url),
+ },
+ allow_redirects=False,
+ timeout=60,
+ )
+ assert response.status_code == 302, "Registration failed"
+ login(client, args.base_url, state)
+ response = client.put(
+ args.base_url + "/settings/api/search.iterations",
+ json={"value": state["value"]},
+ headers={"X-CSRFToken": csrf(client, args.base_url)},
+ timeout=30,
+ )
+ response.raise_for_status()
+ check_saved_setting(client, args.base_url, state["value"])
+ state["cookies"] = client.cookies.get_dict()
+ with args.state.open("x", encoding="utf-8") as target:
+ args.state.chmod(0o600)
+ json.dump(state, target)
+ else:
+ state = json.loads(args.state.read_text(encoding="utf-8"))
+ client.cookies.update(state["cookies"])
+ response = client.get(args.base_url + "/auth/check", timeout=10)
+ assert response.status_code == 401, (
+ "Restart retained an old session"
+ )
+ client.cookies.clear()
+ login(client, args.base_url, state)
+ check_saved_setting(client, args.base_url, state["value"])
+ print(f"Production lifecycle {args.phase}: passed")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/deployment/test_compose_restart_probe.py b/tests/deployment/test_compose_restart_probe.py
new file mode 100644
index 0000000000..475615add0
--- /dev/null
+++ b/tests/deployment/test_compose_restart_probe.py
@@ -0,0 +1,253 @@
+"""Behaviour of the stop/restart probes in compose-integration-test.yml
+(Compose) and docker-tests.yml (docker run).
+
+The step's ``run:`` script is executed with a stub ``docker`` on PATH under
+the shell command GitHub Actions derives from the step's ``shell:`` key
+(``bash -e`` when the key is absent, ``bash --noprofile --norc -eo pipefail``
+for ``shell: bash``), so these tests check which stop grace the probe really
+applies and that it fails closed in the shell CI actually runs.
+"""
+
+# allow: no-sut-import — the code under test is a GitHub Actions workflow step
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+WORKFLOW = REPO_ROOT / ".github" / "workflows" / "compose-integration-test.yml"
+STEP_NAME = "Verify authenticated persistence across Compose restart"
+
+pytestmark = pytest.mark.skipif(
+ shutil.which("bash") is None or shutil.which("jq") is None,
+ reason="needs bash and jq, as on the GitHub runner",
+)
+
+STUB_DOCKER = r"""#!/usr/bin/env bash
+printf '%s\n' "$*" >> "$STUB_LOG"
+case "$*" in
+ "compose config --format json")
+ [ "${STUB_CONFIG_FAIL:-}" = 1 ] && exit 1
+ cat "$STUB_CONFIG_JSON" ;;
+ "compose ps -q local-deep-research") echo stub-cid ;;
+ *"{{.State.ExitCode}}"*) echo "${STUB_EXIT_CODE:-0}" ;;
+ *"{{.State.OOMKilled}}"*) echo "${STUB_OOM_KILLED:-false}" ;;
+ *"{{.State.Error}}"*)
+ [ "${STUB_STATE_ERROR_FAIL:-}" = 1 ] && exit 1
+ echo "${STUB_STATE_ERROR_TEXT:-}" ;;
+esac
+exit 0
+"""
+
+# How GitHub Actions invokes a `run:` script for each `shell:` value
+# (https://docs.github.com/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepsshell).
+# A missing key on a Linux runner means `bash -e {0}`: no pipefail.
+GITHUB_SHELLS = {
+ None: ["bash", "-e"],
+ "bash": ["bash", "--noprofile", "--norc", "-eo", "pipefail"],
+ "sh": ["sh", "-e"],
+}
+
+
+def _step() -> tuple[list[str], str]:
+ """Return the shell argv GitHub would use for the step, and its script."""
+ workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
+ job = workflow["jobs"]["compose-up"]
+ matches = [s for s in job["steps"] if s.get("name") == STEP_NAME]
+ assert len(matches) == 1, f"expected one {STEP_NAME!r} step"
+ step = matches[0]
+ job_default = job.get("defaults", {}).get("run", {}).get("shell")
+ wf_default = workflow.get("defaults", {}).get("run", {}).get("shell")
+ shell = step.get("shell") or job_default or wf_default
+ assert shell in GITHUB_SHELLS, f"unmodelled shell: {shell!r}"
+ return GITHUB_SHELLS[shell], step["run"]
+
+
+def _run(tmp_path: Path, service: dict, **env) -> tuple[int, str, list[str]]:
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ docker = bin_dir / "docker"
+ docker.write_text(STUB_DOCKER, encoding="utf-8")
+ docker.chmod(0o755)
+ config = tmp_path / "config.json"
+ config.write_text(
+ json.dumps({"services": {"local-deep-research": service}}),
+ encoding="utf-8",
+ )
+ log = tmp_path / "docker.log"
+ log.touch()
+ shell_argv, step_script = _step()
+ script = tmp_path / "step.sh"
+ script.write_text(step_script, encoding="utf-8")
+ proc = subprocess.run(
+ [*shell_argv, str(script)],
+ capture_output=True,
+ text=True,
+ timeout=60,
+ env={
+ **os.environ,
+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+ "STUB_LOG": str(log),
+ "STUB_CONFIG_JSON": str(config),
+ **env,
+ },
+ )
+ calls = log.read_text(encoding="utf-8").splitlines()
+ return proc.returncode, proc.stdout + proc.stderr, calls
+
+
+def _stop_calls(calls: list[str]) -> list[str]:
+ return [c for c in calls if c.startswith("compose stop")]
+
+
+def test_declared_stop_grace_period_is_what_the_probe_applies(tmp_path):
+ rc, output, calls = _run(tmp_path, {"stop_grace_period": "1m0s"})
+ assert rc == 0, output
+ stops = _stop_calls(calls)
+ assert len(stops) == 2, calls
+ assert all(c == "compose stop local-deep-research" for c in stops), stops
+ assert "::warning::" not in output
+
+
+def test_missing_stop_grace_period_is_overridden_loudly(tmp_path):
+ rc, output, calls = _run(tmp_path, {})
+ assert rc == 0, output
+ stops = _stop_calls(calls)
+ assert len(stops) == 2, calls
+ assert all(
+ c == "compose stop --timeout 60 local-deep-research" for c in stops
+ ), stops
+ assert output.count("::warning::") == 2
+ assert "stop_grace_period" in output
+ assert "#6918" in output
+
+
[email protected]("exit_code", ["137", "1"])
+def test_forced_or_failed_stop_fails_the_step(tmp_path, exit_code):
+ rc, output, _ = _run(
+ tmp_path, {"stop_grace_period": "1m0s"}, STUB_EXIT_CODE=exit_code
+ )
+ assert rc != 0
+ assert f"Unexpected Compose shutdown exit: {exit_code}" in output
+
+
+def test_unreadable_compose_config_fails_closed(tmp_path):
+ rc, _, calls = _run(tmp_path, {}, STUB_CONFIG_FAIL="1")
+ assert rc != 0
+ assert not _stop_calls(calls)
+
+
+def test_failed_state_error_inspect_fails_closed(tmp_path):
+ """A failing ``docker inspect`` inside ``test -z "$(...)"`` is ignored
+ by ``-e``; the step must not treat it as an empty State.Error."""
+ rc, _, calls = _run(
+ tmp_path, {"stop_grace_period": "1m0s"}, STUB_STATE_ERROR_FAIL="1"
+ )
+ assert rc != 0
+ assert len(_stop_calls(calls)) == 1, calls
+
+
+def _run_native_stop_step(tmp_path: Path, **env) -> tuple[int, str, list[str]]:
+ """Run docker-tests.yml's docker-run stop/restart step with a stub docker."""
+ workflow = yaml.safe_load(
+ (REPO_ROOT / ".github" / "workflows" / "docker-tests.yml").read_text(
+ encoding="utf-8"
+ )
+ )
+ steps = workflow["jobs"]["ldr-production-smoke-test"]["steps"]
+ matches = [
+ s
+ for s in steps
+ if s.get("name") == "Verify persistence and bounded stop/restart"
+ ]
+ assert len(matches) == 1
+ assert matches[0].get("shell") == "bash"
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ docker = bin_dir / "docker"
+ docker.write_text(STUB_DOCKER, encoding="utf-8")
+ docker.chmod(0o755)
+ log = tmp_path / "docker.log"
+ log.touch()
+ script = tmp_path / "step.sh"
+ script.write_text(matches[0]["run"], encoding="utf-8")
+ proc = subprocess.run(
+ [*GITHUB_SHELLS["bash"], str(script)],
+ capture_output=True,
+ text=True,
+ timeout=60,
+ env={
+ **os.environ,
+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+ "STUB_LOG": str(log),
+ **env,
+ },
+ )
+ calls = log.read_text(encoding="utf-8").splitlines()
+ return proc.returncode, proc.stdout + proc.stderr, calls
+
+
+def _native_stops(calls: list[str]) -> list[str]:
+ return [c for c in calls if c.startswith("stop")]
+
+
+def test_docker_run_smoke_warns_that_it_overrides_the_default_stop_grace(
+ tmp_path,
+):
+ """The docker-run smoke stops with --timeout 60 while README users get
+ Docker's 10s default (#6918); it must say so like the Compose step."""
+ rc, output, calls = _run_native_stop_step(tmp_path)
+ assert rc == 0, output
+ assert _native_stops(calls) == ["stop --timeout 60 ldr-prod-test"] * 2, (
+ calls
+ )
+ assert output.count("::warning::") == 2
+ assert "#6918" in output
+
+
[email protected]("exit_code", ["0", "143"])
+def test_docker_run_smoke_accepts_graceful_exit_codes(tmp_path, exit_code):
+ rc, output, calls = _run_native_stop_step(
+ tmp_path, STUB_EXIT_CODE=exit_code
+ )
+ assert rc == 0, output
+ assert len(_native_stops(calls)) == 2, calls
+
+
[email protected]("exit_code", ["137", "1"])
+def test_docker_run_smoke_forced_or_failed_stop_fails_the_step(
+ tmp_path, exit_code
+):
+ rc, output, calls = _run_native_stop_step(
+ tmp_path, STUB_EXIT_CODE=exit_code
+ )
+ assert rc != 0
+ assert f"Unexpected shutdown exit: {exit_code}" in output
+ assert len(_native_stops(calls)) == 1, calls
+
+
+def test_docker_run_smoke_oom_kill_fails_the_step(tmp_path):
+ rc, _, calls = _run_native_stop_step(tmp_path, STUB_OOM_KILLED="true")
+ assert rc != 0
+ assert len(_native_stops(calls)) == 1, calls
+
+
+def test_docker_run_smoke_failed_state_error_inspect_fails_closed(tmp_path):
+ """A failing ``docker inspect`` inside ``test -z "$(...)"`` is ignored
+ by ``-e``; the step must not treat it as an empty State.Error."""
+ rc, _, calls = _run_native_stop_step(tmp_path, STUB_STATE_ERROR_FAIL="1")
+ assert rc != 0
+ assert len(_native_stops(calls)) == 1, calls
+
+
+def test_docker_run_smoke_non_empty_state_error_fails_the_step(tmp_path):
+ rc, _, calls = _run_native_stop_step(tmp_path, STUB_STATE_ERROR_TEXT="boom")
+ assert rc != 0
+ assert len(_native_stops(calls)) == 1, calls