Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Retain OpenRC agent recovery until removal is confirmed

pulse-triage[bot]174f440e2d17
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index cabef9e7e3..467fa1f357 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -15,6 +15,32 @@
## Purpose
+### OpenRC removal confirms stopped and unlinked state before erasure
+
+OpenRC removal must observe a complete, diagnostic-free runlevel inventory and
+a matching bounded service status before remote deregistration or local erasure.
+Started services require a successful stop; every configured runlevel, including
+operator-selected levels other than `default`, must be disabled. Independent
+inventory and status readback must confirm no runlevel activation and a stopped
+service. A status exit alone is insufficient: the matching colour-free status
+line is required. Failed, empty, duplicate, malformed, diagnostic-bearing or
+unknown observations refuse without obtaining a replacement sample.
+
+After confirmed server removal, OpenRC teardown rechecks these conditions before
+removing its init script, process state or recovery directories. A late refusal
+can therefore follow remote removal, but must still retain all local recovery
+files; it does not restore a previously stopped service. Missing init scripts
+require both manager absence and an independent silent process-absence result.
+Linked, non-regular or non-executable init paths cannot authorise teardown.
+
+`TestInstallSHOpenRCTeardownRequiresObservedQuiescence` runs the actual production
+functions and initial uninstall caller through `agent_openrc_teardown.py`, using
+closed OpenRC/transport/process models and only owned temporary deletion targets.
+It requires every refusal, final-readback, custom-runlevel, late-refusal, no-op,
+path and ordering control, plus a real timeout that stops its owned child.
+These are confined source controls, not native OpenRC, signed installed-agent,
+real TLS, Windows, other service-manager or whole-issue #1099 acceptance.
+
### Systemd agent removal retains recovery material until quiescence is observed
Unix agent removal must confirm the collector, action runner and privileged
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 0877bcf585..2ac6f51014 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -16,6 +16,27 @@
## Purpose
+### OpenRC removal retains recovery on an unconfirmed manager result
+
+The OpenRC agent-removal path must finish stop and all-runlevel disable with
+independent status/inventory readback before server deregistration or local
+mutation. A failed, partial, diagnostic-bearing or unknown observation cannot
+be treated as absence or retried for a more favourable sample. Teardown rechecks
+after the remote operation, before any recovery directory is deleted, so a late
+manager refusal retains local recovery even if remote removal completed.
+Known absent/repeated removal requires manager and process-absence evidence;
+no unrelated init file, service or runlevel is removed. No failed partial stop
+is restarted automatically and no credential or transport authority changes.
+
+The registered `scripts/installtests/install_sh_test.go` artifact requires all
+twelve `agent_openrc_teardown.py` production-function/caller controls, including
+custom runlevels, stopped/refused/unknown states, post-remote refusal, exact
+cleanup ordering, path exclusions, and real bounded timeout-child termination.
+The manager and transport are closed models, with deletion confined to owned
+private files. Required containing CI, native OpenRC compatibility, real saved
+install/TLS, Windows and other-manager acceptance remain separate; this does
+not establish published or installed Linux/PowerShell #1099 recovery.
+
### Linux agent removal native boundary
The existing Unified Agent Native Verification job now runs the unchanged Unix
diff --git a/scripts/install.sh b/scripts/install.sh
index 601f7be565..4bd96fec8e 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -1237,10 +1237,104 @@ teardown_action_runner_service() {
rm -rf "$ACTION_RUNNER_CONFIG_DIR" "$ACTION_RUNNER_STATE_DIR"
}
+# OpenRC's status command uses 3 for a stopped service. Neither that exit
+# alone nor a failed manager read proves quiescence: retain the complete,
+# colour-free response and require the matching status line as well.
+read_openrc_agent_removal_status() {
+ local observation="" line="" status="" result="" status_count=0 result_count=0
+ observation=$(LC_ALL=C RC_NOCOLOR=yes timeout -k 1 5 rc-service "${AGENT_NAME}" status 2>&1 &&
+ printf '%s' '__pulse_openrc_status_exit__=0' || printf '%s' "__pulse_openrc_status_exit__=$?") || return 1
+ while IFS= read -r line; do
+ [[ $result_count -eq 0 ]] || return 1
+ if [[ "$line" =~ ^[[:space:]]*\*[[:space:]]+status:[[:space:]]+(started|stopped)[[:space:]]*$ ]]; then
+ status="${BASH_REMATCH[1]}"
+ status_count=$((status_count + 1))
+ elif [[ "$line" =~ ^__pulse_openrc_status_exit__=([0-9]+)$ ]]; then
+ result="${BASH_REMATCH[1]}"
+ result_count=$((result_count + 1))
+ else
+ return 1
+ fi
+ done <<< "$observation"
+ [[ $status_count -eq 1 && $result_count -eq 1 ]] || return 1
+ if [[ "$status" == started ]]; then
+ [[ "$result" == 0 ]] || return 1
+ else
+ [[ "$result" == 0 || "$result" == 3 ]] || return 1
+ fi
+ printf '%s\n' "$status"
+}
+
+read_openrc_agent_removal_runlevels() {
+ local observation="" line="" name="" levels="" level="" seen=" " count=0 complete="false"
+ local IFS=$' \t\n'
+ local -a row_levels=()
+ observation=$(LC_ALL=C RC_NOCOLOR=yes timeout -k 1 5 rc-update show -v 2>&1 &&
+ printf '%s' '__pulse_openrc_runlevels_complete__') || return 1
+ while IFS= read -r line; do
+ [[ "$complete" == false ]] || return 1
+ if [[ "$line" == __pulse_openrc_runlevels_complete__ ]]; then
+ complete="true"
+ elif [[ "$line" =~ ^[[:space:]]*([A-Za-z0-9][A-Za-z0-9_.-]*)[[:space:]]+\|[[:space:]]*(.*)$ ]]; then
+ name="${BASH_REMATCH[1]}"
+ levels="${BASH_REMATCH[2]}"
+ read -r -a row_levels <<< "$levels"
+ # Validate every row, not just a matching substring of diagnostics.
+ for level in "${row_levels[@]}"; do
+ [[ "$level" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ && ${#level} -le 64 ]] || return 1
+ done
+ if [[ "$name" == "$AGENT_NAME" ]]; then
+ count=$((count + 1))
+ [[ $count -eq 1 ]] || return 1
+ for level in "${row_levels[@]}"; do
+ [[ "$seen" != *" $level "* ]] || return 1
+ seen+="$level "
+ done
+ fi
+ else
+ return 1
+ fi
+ done <<< "$observation"
+ [[ "$complete" == true ]] || return 1
+ if [[ $count -eq 0 ]]; then
+ printf 'absent\n'
+ else
+ printf 'present%s\n' "${seen% }"
+ fi
+}
+
+quiesce_openrc_agent_for_removal() {
+ local init_path="${1:-/etc/init.d/${AGENT_NAME}}" status="" levels="" observation=""
+ # A missing script is a no-op only with a complete manager inventory and
+ # an independent, successful process-absence read. Unknown is not absent.
+ levels=$(read_openrc_agent_removal_runlevels) || return 1
+ if [[ ! -e "$init_path" && ! -L "$init_path" ]]; then
+ [[ "$levels" == absent ]] || return 1
+ observation=$(timeout -k 1 5 pgrep -x "${AGENT_NAME}" 2>&1 &&
+ printf '%s' '__pulse_openrc_process_exit__=0' || printf '%s' "__pulse_openrc_process_exit__=$?") || return 1
+ [[ "$observation" == __pulse_openrc_process_exit__=1 ]] || return 1
+ return 0
+ fi
+ [[ -f "$init_path" && ! -L "$init_path" && -x "$init_path" && "$levels" == present* ]] || return 1
+ status=$(read_openrc_agent_removal_status) || return 1
+ if [[ "$status" != stopped ]]; then
+ LC_ALL=C RC_NOCOLOR=yes timeout -k 1 5 rc-service "${AGENT_NAME}" stop >/dev/null 2>&1 || return 1
+ fi
+ if [[ "$levels" != present ]]; then
+ # Custom runlevels can restart an installation too; deleting only the
+ # installer's default link is not sufficient for destructive removal.
+ LC_ALL=C RC_NOCOLOR=yes timeout -k 1 5 rc-update del "${AGENT_NAME}" --all >/dev/null 2>&1 || return 1
+ fi
+ levels=$(read_openrc_agent_removal_runlevels) || return 1
+ [[ "$levels" == present ]] || return 1
+ status=$(read_openrc_agent_removal_status) || return 1
+ [[ "$status" == stopped ]]
+}
+
teardown_openrc_agent_service() {
local init_path="${1:-/etc/init.d/${AGENT_NAME}}"
- rc-service "${AGENT_NAME}" stop 2>/dev/null || true
- rc-update del "${AGENT_NAME}" default 2>/dev/null || true
+ quiesce_openrc_agent_for_removal "$init_path" ||
+ fail "OpenRC removal could not confirm a stopped service with no runlevel activation; credentials and files were retained. Reconcile service state before retrying; no automatic restart was attempted." "$EXIT_GENERAL"
rm -f "$init_path"
}
@@ -5609,6 +5703,10 @@ if [[ "$UNINSTALL" == "true" ]]; then
"${PRIVILEGED_HELPER_NAME}.socket" "${PRIVILEGED_HELPER_NAME}.service" "${AGENT_NAME}.service" ||
fail "Agent removal could not confirm every systemd unit stopped and disabled; local credentials and files were retained. Reconcile service state before retrying; no automatic restart was attempted." "$EXIT_GENERAL"
fi
+ if command -v rc-service >/dev/null 2>&1; then
+ quiesce_openrc_agent_for_removal ||
+ fail "Agent removal could not confirm OpenRC stopped with no runlevel activation; local credentials and files were retained. A preceding stop or disable may have succeeded: reconcile service state before retrying; no automatic restart was attempted." "$EXIT_GENERAL"
+ fi
# A credential-bearing install must durably remove its exact server record
# through the same CA/fingerprint/no-proxy transport used by lifecycle
@@ -5622,6 +5720,12 @@ if [[ "$UNINSTALL" == "true" ]]; then
elif [[ -n "$PULSE_URL" ]]; then
log_warn "No local collector credential exists; continuing with local-only removal."
fi
+ # Reconcile OpenRC once more after the remote operation, while all local
+ # recovery state still exists. A late refusal must not happen only after
+ # remove_agent_state_dir has already erased that state.
+ if command -v rc-service >/dev/null 2>&1; then
+ teardown_openrc_agent_service
+ fi
# Kill wrapper scripts first: they are watchdogs, so stopping the agent
# while its wrapper still loops only races the respawn. No leading path
@@ -5759,11 +5863,6 @@ if [[ "$UNINSTALL" == "true" ]]; then
rm -rf "$TRUENAS_STATE_DIR"
fi
- # OpenRC (Alpine, Gentoo, Artix, etc.)
- if command -v rc-service >/dev/null 2>&1; then
- teardown_openrc_agent_service
- fi
-
# Vanilla FreeBSD, OPNsense, and pfSense all use the rc.d service rendered
# by this installer. This must run even when no TrueNAS marker is present.
if [[ "$(uname -s)" == "FreeBSD" ]]; then
diff --git a/scripts/installtests/agent_openrc_teardown.py b/scripts/installtests/agent_openrc_teardown.py
new file mode 100644
index 0000000000..a2cc349ea5
--- /dev/null
+++ b/scripts/installtests/agent_openrc_teardown.py
@@ -0,0 +1,374 @@
+#!/usr/bin/env python3
+"""Execute production OpenRC removal with closed, ordinary-user manager models.
+
+Only private fixture files can be deleted. The full caller's initial admission
+and cleanup run too, with transport/process/state operations replaced. Paths are
+redirected to the fixture; there is no privileged installer, native OpenRC,
+published installation, real credential or remote contact in this proof.
+"""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import shlex
+import stat
+import subprocess
+import tempfile
+import time
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SOURCE = Path(os.environ.get("PULSE_AGENT_OPENRC_SOURCE", ROOT / "scripts/install.sh"))
+FUNCTIONS = ("read_openrc_agent_removal_status", "read_openrc_agent_removal_runlevels",
+ "quiesce_openrc_agent_for_removal", "teardown_openrc_agent_service")
+
+
+def definitions(source):
+ # Parent adverse controls execute its existing teardown/caller as-is, not
+ # a substitute implementation of the newly added admission functions.
+ selected = []
+ for name in FUNCTIONS:
+ marker = name + "() {"
+ if marker not in source:
+ continue
+ if source.count(marker) != 1:
+ raise ValueError("ambiguous production function")
+ selected.append(marker + source.split(marker, 1)[1].split("\n}\n", 1)[0] + "\n}\n")
+ return "\n".join(selected)
+
+
+MODEL = r'''#!/usr/bin/python3
+import json, os, pathlib, sys
+root = pathlib.Path(os.environ['FIXTURE'])
+model = root / 'model.json'
+value = json.loads(model.read_text())
+role, args = pathlib.Path(sys.argv[0]).name, sys.argv[1:]
+fault = value['fault']
+remote = (root / 'events').exists() and 'remote:remove' in (root / 'events').read_text()
+def save(): model.write_text(json.dumps(value))
+def event(text):
+ with (root / 'events').open('a') as out: out.write(text + '\n')
+if role == 'rc-service':
+ assert len(args) == 2 and args[0] == 'pulse-agent'
+ verb = args[1]
+ event('service:' + verb)
+ if verb == 'stop':
+ if fault == 'refused-stop' or (fault == 'refused-stop-after-remote' and remote): sys.exit(1)
+ if fault != 'ignored-stop': value['state'] = 'stopped'
+ save(); sys.exit(0)
+ assert verb == 'status'
+ value['status_reads'] += 1; save()
+ if fault == 'timeout-status':
+ (root / 'timed-pid').write_text(str(os.getpid()))
+ os.execl('/bin/sleep', 'sleep', '30')
+ state = value['state']
+ code = 0 if state == 'started' else 3
+ if fault == 'empty-status': sys.exit(code)
+ if fault == 'unknown-status': state = 'crashed'
+ if fault == 'failed-status': code = 2
+ if fault == 'wrong-started-exit': code = 3
+ if fault == 'failed-final-status' and value['status_reads'] > 1: code = 2
+ if fault == 'failed-status-after-remote' and remote: code = 2
+ print(' * status: ' + state)
+ if fault == 'duplicate-status': print(' * status: ' + state)
+ if fault == 'blank-status': print()
+ if fault == 'diagnostic-status': print('manager connection refused', file=sys.stderr)
+ sys.exit(code)
+elif role == 'rc-update':
+ if args == ['show', '-v']:
+ event('runlevels:show')
+ value['inventory_reads'] += 1; save()
+ final = value['inventory_reads'] > 1
+ if fault == 'empty-inventory': sys.exit(0)
+ if fault == 'failed-inventory' or (fault == 'failed-final-inventory' and final): sys.exit(2)
+ print(' unrelated | boot')
+ if (root / 'init').exists() or fault == 'missing-script-listed':
+ levels = value['levels']
+ if fault == 'invalid-level': levels = ['default*']
+ if fault == 'duplicate-level': levels = ['default', 'default']
+ print(' pulse-agent | ' + ' '.join(levels))
+ if fault == 'duplicate-inventory': print(' pulse-agent | default')
+ if fault == 'blank-inventory': print()
+ if fault == 'diagnostic-inventory': print('manager connection refused', file=sys.stderr)
+ sys.exit(0)
+ assert args in (['del', 'pulse-agent', '--all'], ['del', 'pulse-agent', 'default']), args
+ event('runlevels:disable-all' if args[-1] == '--all' else 'runlevels:disable-default')
+ if fault == 'refused-disable': sys.exit(1)
+ if fault != 'ignored-disable':
+ value['levels'] = [] if args[-1] == '--all' else [v for v in value['levels'] if v != 'default']
+ if fault == 'reactivation': value['state'] = 'started'
+ save(); sys.exit(0)
+elif role == 'pgrep':
+ assert args == ['-x', 'pulse-agent']
+ event('process:observe')
+ if fault == 'orphan-process': print('12345'); sys.exit(0)
+ if fault == 'failed-process': sys.exit(2)
+ if fault == 'diagnostic-process': print('read refused', file=sys.stderr)
+ if fault == 'blank-process': print()
+ sys.exit(1)
+else: raise ValueError('operation outside fixture')
+'''
+
+
+class AgentOpenRCTeardown(unittest.TestCase):
+ source = SOURCE.read_text()
+
+ def exercise(self, *, fault="", flow="teardown", state="started", levels=("default", "custom"),
+ path="regular", repeat=False, reply="confirmed"):
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ tools = root / "tools"
+ tools.mkdir()
+ for name in ("rc-service", "rc-update", "pgrep"):
+ tool = tools / name
+ tool.write_text(MODEL)
+ tool.chmod(0o700)
+ init = root / "init"
+ unrelated = root / "unrelated"
+ unrelated.write_text("unrelated recovery file\n")
+ if path == "linked":
+ init.symlink_to(unrelated)
+ elif path == "directory":
+ init.mkdir()
+ elif path != "missing":
+ init.write_text("owned service script\n")
+ init.chmod(0o600 if path == "not-executable" else 0o700)
+ for name in ("state", "helper", "profile"):
+ directory = root / name
+ directory.mkdir()
+ (directory / "token").write_text("synthetic recovery material\n")
+ # A glob must never turn an invalid runlevel into this valid name.
+ (root / "default-other").write_text("not a runlevel\n")
+ model = dict(state=state, levels=list(levels), fault=fault, inventory_reads=0, status_reads=0)
+ (root / "model.json").write_text(json.dumps(model))
+ before = self.snapshot(root)
+ values = dict(INIT_PATH=init, STATE_DIR=root / "state", PRIVILEGED_HELPER_STATE_DIR=root / "helper",
+ SAFE_PROFILE_STATE_DIR=root / "profile", EVENTS=root / "events", REPLY=reply)
+ prefix = "\n".join(k + "=" + shlex.quote(str(v)) for k, v in values.items()) + "\n"
+ shell = r'''
+set -euo pipefail
+AGENT_NAME=pulse-agent UNINSTALL=true EXIT_GENERAL=1 EXIT_MISSING_ARGS=16
+PULSE_URL=https://same-instance.example.invalid AGENT_ID=synthetic
+ACTION_RUNNER_NAME=pulse-agent-runner PRIVILEGED_HELPER_NAME=pulse-agent-helper
+log_info() { :; }
+log_warn() { :; }
+fail() { printf '%s\n' "$1" >&2; exit "${2:-1}"; }
+command() {
+ if [[ "$*" == '-v systemctl' ]]; then return 1; fi
+ builtin command "$@"
+}
+# No operation below can mutate a service, process or non-fixture path.
+collector_credential_state_present() { return 0; }
+uninstall_collector_registration() {
+ printf 'remote:remove\n' >> "$EVENTS"
+ python3 - <<'END'
+import json, os, pathlib
+path = pathlib.Path(os.environ['FIXTURE']) / 'model.json'
+value = json.loads(path.read_text())
+if value['fault'] == 'refused-stop-after-remote':
+ value['state'], value['levels'] = 'started', ['custom']
+ path.write_text(json.dumps(value))
+END
+ [[ "$REPLY" == confirmed ]]
+}
+pkill() { printf 'process:stop\n' >> "$EVENTS"; }
+sleep() { :; }
+rm() {
+ [[ $# == 2 && "$1" == -f && "$2" == "$INIT_PATH" ]] || return 97
+ printf 'init:remove\n' >> "$EVENTS"
+ builtin command rm "$@"
+}
+remove_agent_state_dir() {
+ [[ "$1" == "$STATE_DIR" ]] || return 96
+ printf 'state:remove\n' >> "$EVENTS"
+ builtin command rm -r -- "$STATE_DIR"
+}
+remove_privileged_helper_state_dir() { builtin command rm -r -- "$PRIVILEGED_HELPER_STATE_DIR"; }
+remove_safe_profile_state_dir() { builtin command rm -r -- "$SAFE_PROFILE_STATE_DIR"; }
+'''
+ if flow == "caller":
+ entry = self.source.split("# --- Uninstall Logic ---\n", 1)[1].split(
+ " # Remove least-privilege helper artifacts.", 1)[0] + "fi\n"
+ # Redirect only the fixed init path, as the fixture does with
+ # STATE_DIR. The actual caller/production function bodies run.
+ entry = entry.replace("quiesce_openrc_agent_for_removal ||",
+ 'quiesce_openrc_agent_for_removal "$INIT_PATH" ||')
+ entry = entry.replace(" teardown_openrc_agent_service\n",
+ ' teardown_openrc_agent_service "$INIT_PATH"\n')
+ invocation = "main() {\n" + entry + "}\nmain\n"
+ elif flow == "status":
+ invocation = "read_openrc_agent_removal_status\n"
+ elif flow == "inventory":
+ invocation = "read_openrc_agent_removal_runlevels\n"
+ else:
+ invocation = 'teardown_openrc_agent_service "$INIT_PATH"\n' * (2 if repeat else 1)
+ env = dict(os.environ, PATH=str(tools) + ":" + os.environ["PATH"], FIXTURE=str(root))
+ result = subprocess.run(["bash", "-c", prefix + shell + definitions(self.source) + invocation],
+ cwd=root, env=env, capture_output=True, text=True, timeout=10)
+ events = (root / "events").read_text().splitlines() if (root / "events").exists() else []
+ model = json.loads((root / "model.json").read_text())
+ if (root / "timed-pid").exists():
+ pid = int((root / "timed-pid").read_text())
+ try:
+ os.kill(pid, 0)
+ model["timeout_child_absent"] = False
+ except ProcessLookupError:
+ model["timeout_child_absent"] = True
+ return result, events, before, self.snapshot(root), model
+
+ @staticmethod
+ def snapshot(root):
+ result = {}
+ for name in ("init", "state", "helper", "profile", "unrelated", "default-other"):
+ path = root / name
+ paths = [path] + (list(path.rglob("*")) if path.is_dir() else [])
+ for item in paths:
+ if not item.exists() and not item.is_symlink():
+ continue
+ info = item.lstat()
+ payload = (os.readlink(item) if stat.S_ISLNK(info.st_mode) else
+ item.read_bytes() if stat.S_ISREG(info.st_mode) else None)
+ result[str(item.relative_to(root))] = (info.st_mode, info.st_ino, payload)
+ return result
+
+ def test_stop_and_disable_refusals_retain_every_local_file(self):
+ for fault in ("refused-stop", "refused-disable", "ignored-stop", "ignored-disable", "reactivation"):
+ with self.subTest(fault=fault):
+ result, events, before, after, _ = self.exercise(fault=fault)
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+ self.assertNotIn("init:remove", events)
+
+ def test_failed_partial_or_diagnostic_observations_are_not_replaced_by_another_read(self):
+ for fault in ("failed-inventory", "empty-inventory", "duplicate-inventory", "blank-inventory",
+ "diagnostic-inventory", "invalid-level", "duplicate-level", "failed-status", "empty-status",
+ "unknown-status", "duplicate-status", "blank-status", "diagnostic-status", "wrong-started-exit"):
+ with self.subTest(fault=fault):
+ result, events, before, after, model = self.exercise(fault=fault)
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+ self.assertNotIn("init:remove", events)
+ self.assertLessEqual(model["inventory_reads"], 1)
+ self.assertLessEqual(model["status_reads"], 1)
+
+ def test_independent_post_stop_and_runlevel_reads_are_mandatory(self):
+ for fault in ("failed-final-status", "failed-final-inventory", "ignored-disable", "reactivation"):
+ with self.subTest(fault=fault):
+ result, events, before, after, _ = self.exercise(fault=fault)
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+ self.assertNotIn("init:remove", events)
+
+ def test_refused_manager_cannot_start_remote_or_local_removal(self):
+ for fault in ("refused-stop", "refused-disable", "failed-status", "failed-inventory", "reactivation"):
+ with self.subTest(fault=fault):
+ result, events, before, after, _ = self.exercise(fault=fault, flow="caller")
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+ for forbidden in ("remote:remove", "process:stop", "state:remove"):
+ self.assertNotIn(forbidden, events)
+
+ def test_quiescence_precedes_server_process_and_state_mutation(self):
+ result, events, _, after, model = self.exercise(flow="caller")
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertLess(events.index("runlevels:disable-all"), events.index("remote:remove"))
+ observations = [i for i, e in enumerate(events) if e == "service:status"]
+ self.assertEqual(len(observations), 4)
+ self.assertLess(observations[1], events.index("remote:remove"))
+ self.assertLess(events.index("remote:remove"), observations[2])
+ self.assertLess(observations[3], events.index("init:remove"))
+ self.assertLess(events.index("init:remove"), events.index("state:remove"))
+ self.assertLess(events.index("remote:remove"), events.index("state:remove"))
+ self.assertNotIn("state/token", after)
+ self.assertEqual(model["state"], "stopped")
+ self.assertEqual(model["levels"], [])
+
+ def test_server_refusal_keeps_stopped_disabled_recovery_without_restart(self):
+ result, events, before, after, model = self.exercise(flow="caller", reply="refused")
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+ self.assertEqual(model["state"], "stopped")
+ self.assertEqual(model["levels"], [])
+ self.assertNotIn("state:remove", events)
+ self.assertNotIn("process:stop", events)
+
+ def test_late_manager_refusal_keeps_recovery_after_confirmed_remote_removal(self):
+ for fault in ("failed-status-after-remote", "refused-stop-after-remote"):
+ with self.subTest(fault=fault):
+ result, events, before, after, _ = self.exercise(flow="caller", fault=fault)
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+ self.assertEqual(events.count("remote:remove"), 1)
+ for forbidden in ("state:remove", "process:stop", "init:remove"):
+ self.assertNotIn(forbidden, events)
+
+ def test_known_stopped_state_and_every_runlevel_allow_only_selected_teardown(self):
+ for state, levels in (("started", ("default",)), ("started", ("custom", "boot", "default")),
+ ("stopped", ("custom",)), ("stopped", ())):
+ with self.subTest(state=state, levels=levels):
+ result, events, before, after, model = self.exercise(state=state, levels=levels)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, {k: v for k, v in before.items() if k != "init"})
+ self.assertEqual(model["state"], "stopped")
+ self.assertEqual(model["levels"], [])
+ self.assertEqual(events.count("service:stop"), int(state == "started"))
+ self.assertEqual(events.count("runlevels:disable-all"), int(bool(levels)))
+
+ def test_absent_and_repeated_removal_need_manager_and_process_absence(self):
+ result, events, before, after, _ = self.exercise(path="missing")
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+ self.assertIn("process:observe", events)
+ result, events, _, _, _ = self.exercise(repeat=True)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(events.count("service:stop"), 1)
+ self.assertEqual(events.count("runlevels:disable-all"), 1)
+ for fault in ("missing-script-listed", "orphan-process", "failed-process", "diagnostic-process", "blank-process"):
+ with self.subTest(fault=fault):
+ result, _, before, after, _ = self.exercise(path="missing", fault=fault)
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+
+ def test_unexpected_service_paths_refuse_before_stop_or_deletion(self):
+ for path in ("linked", "directory", "not-executable"):
+ with self.subTest(path=path):
+ result, events, before, after, _ = self.exercise(path=path)
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(after, before)
+ self.assertNotIn("service:stop", events)
+ self.assertNotIn("init:remove", events)
+
+ def test_source_selection_and_caller_order_have_one_bounded_observation_per_operation(self):
+ selected = definitions(self.source)
+ for name in FUNCTIONS:
+ self.assertEqual(selected.count(name + "() {"), 1)
+ for command in ("rc-service", "rc-update", "pgrep"):
+ self.assertNotRegex(selected, r"(?m)^\s*" + command + r"\s")
+ self.assertIn('timeout -k 1 5 rc-service', selected)
+ self.assertIn('timeout -k 1 5 rc-update', selected)
+ self.assertIn('timeout -k 1 5 pgrep', selected)
+ entry = self.source.split("# --- Uninstall Logic ---\n", 1)[1]
+ admission = entry.index("quiesce_openrc_agent_for_removal ||")
+ for later in ("if ! uninstall_collector_registration;", "pkill -f", 'remove_agent_state_dir "$STATE_DIR"'):
+ self.assertLess(admission, entry.index(later))
+ teardown = entry.index(" teardown_openrc_agent_service\n")
+ self.assertLess(entry.index("if ! uninstall_collector_registration;"), teardown)
+ self.assertLess(teardown, entry.index('remove_agent_state_dir "$STATE_DIR"'))
+ print("OPENRC_SOURCE_SHA256=" + hashlib.sha256(self.source.encode()).hexdigest())
+ print("OPENRC_DEFINITIONS_SHA256=" + hashlib.sha256(selected.encode()).hexdigest())
+
+ def test_real_status_timeout_stops_its_owned_child_without_erasure(self):
+ started = time.monotonic()
+ result, events, before, after, model = self.exercise(fault="timeout-status")
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(before, after)
+ self.assertNotIn("init:remove", events)
+ self.assertTrue(model["timeout_child_absent"])
+ self.assertGreaterEqual(time.monotonic() - started, 4.8)
+ self.assertLess(time.monotonic() - started, 9)
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go
index c90e433b7c..9b49b16c56 100644
--- a/scripts/installtests/install_sh_test.go
+++ b/scripts/installtests/install_sh_test.go
@@ -16,6 +16,7 @@ import (
"os/exec"
"path/filepath"
"regexp"
+ "runtime"
"strconv"
"strings"
"sync"
@@ -8372,3 +8373,38 @@ func TestInstallSHSystemdTeardownRequiresObservedQuiescence(t *testing.T) {
}
t.Logf("%s", output)
}
+
+func TestInstallSHOpenRCTeardownRequiresObservedQuiescence(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("Unix shell/OpenRC controls; Windows has its own native SCM gate")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+ defer cancel()
+ cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_openrc_teardown.py"))
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("OpenRC teardown refusal controls: %v\n%s", err, output)
+ }
+ for _, name := range []string{
+ "test_stop_and_disable_refusals_retain_every_local_file",
+ "test_failed_partial_or_diagnostic_observations_are_not_replaced_by_another_read",
+ "test_independent_post_stop_and_runlevel_reads_are_mandatory",
+ "test_refused_manager_cannot_start_remote_or_local_removal",
+ "test_quiescence_precedes_server_process_and_state_mutation",
+ "test_server_refusal_keeps_stopped_disabled_recovery_without_restart",
+ "test_late_manager_refusal_keeps_recovery_after_confirmed_remote_removal",
+ "test_known_stopped_state_and_every_runlevel_allow_only_selected_teardown",
+ "test_absent_and_repeated_removal_need_manager_and_process_absence",
+ "test_unexpected_service_paths_refuse_before_stop_or_deletion",
+ "test_source_selection_and_caller_order_have_one_bounded_observation_per_operation",
+ "test_real_status_timeout_stops_its_owned_child_without_erasure",
+ } {
+ if !strings.Contains(string(output), name+" (__main__.AgentOpenRCTeardown."+name+") ... ok") {
+ t.Fatalf("OpenRC teardown control %s did not pass:\n%s", name, output)
+ }
+ }
+ if !strings.Contains(string(output), "\nRan 12 tests in ") || !strings.Contains(string(output), "\nOK\n") {
+ t.Fatalf("incomplete OpenRC teardown verdict:\n%s", output)
+ }
+ t.Logf("%s", output)
+}