Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

44.9 KB
Raw Blame
#!/usr/bin/env python3
"""Actual NAS uninstall caller and boot transformations in private user fixtures.

Flash and kernel/manager refusal controls are models. No host signal, mount,
service, credential, boot path, network or native appliance is exercised.
"""
import json
import os
from pathlib import Path
import shlex
import shutil
import stat
import subprocess
import sys
import tempfile
import time
import unittest
from unittest import mock

ROOT = Path(__file__).resolve().parents[2]
SOURCE = Path(os.environ.get("PULSE_AGENT_NAS_SOURCE", ROOT / "scripts/install.sh"))
FUNCTIONS = (
    "portable_path_uid", "portable_path_mode", "portable_sed_in_place",
    "nas_agent_removal_platform", "nas_agent_removal_file_safe", "nas_agent_removal_paths_safe",
    "filter_nas_agent_boot_file", "disable_nas_agent_boot_file", "nas_agent_boot_operation",
    "read_nas_agent_removal_processes", "nas_agent_processes_absent", "quiesce_nas_agent_for_removal",
    "confirm_nas_agent_removal_quiescent", "teardown_nas_agent_boot_owner",
    "render_qnap_autorun_block", "append_qnap_autorun_block",
    "render_unraid_go_launch", "write_unraid_go_entry",
)


def definitions(source, names=FUNCTIONS):
    result = []
    for name in names:
        marker = name + "() {"
        if marker not in source:  # Exact-parent adverse controls, not substitute code.
            continue
        if source.count(marker) != 1:
            raise ValueError("ambiguous production function " + name)
        if name == "write_qnap_wrapper_script":
            # Its heredoc contains shell functions: their braces do not close
            # the renderer. Bind the actual outer EOF/function boundary.
            start = source.index(marker)
            end = source.index("\n}\n", source.index("\nEOF\n", start)) + 3
            result.append(source[start:end] + "\n")
        else:
            result.append(marker + source.split(marker, 1)[1].split("\n}\n", 1)[0] + "\n}\n")
    return "\n".join(result)


def generated_collector_is_running(listing, runtime):
    """Match the owned executable with either Unix comm representation.

    Darwin reports an executable path where Linux reports its basename. Keep
    the argument path exact so another same-named process is not our collector.
    """
    for line in listing.splitlines():
        fields = line.split(maxsplit=1)
        if (len(fields) == 2 and Path(fields[0]).name == "pulse-agent"
                and fields[1].startswith(str(runtime) + " ")):
            return True
    return False


def write_wide_ps_adapter(tools, executable):
    """Observe real fixture processes with a native basename and full argv.

    These Linux NAS wrappers run in a long private Darwin temporary path during
    CI. Darwin truncates a middle comm path even with -ww. Its native ucomm
    field supplies the basename Linux's comm uses; args still supplies the
    exact executable path. Change only this fixture transport, not the
    production parser, ownership checks or deadlines.
    """
    target = tools / "ps"
    target.write_text("#!/bin/sh\n" + r'''
for arg do
    case "$arg" in comm=) arg=ucomm= ;; esac
    set -- "$@" "$arg"
    shift
done
exec ''' + shlex.quote(str(executable)) + ' -ww "$@"\n')
    target.chmod(0o700)
    return target


MODEL = r'''
import json, os, pathlib, sys
root = pathlib.Path(os.environ['FIXTURE'])
path = root / 'model.json'
v = json.loads(path.read_text()); role, args = sys.argv[1], sys.argv[2:]
fault = v['fault']
def event(s):
    with (root / 'events').open('a') as f: f.write(s + '\n')
def save():
    pending = root / 'model.pending'; pending.write_text(json.dumps(v)); pending.replace(path)
if role == 'ps':
    assert args == ['-ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'comm=', '-o', 'args='], args
    event('process:read'); v['reads'] += 1
    if fault == 'reactivated-before-remote' and v['reads'] == 5: v['watchdog'] = True
    save()
    if fault == 'failed-process': sys.exit(2)
    if fault == 'empty-process': sys.exit(0)
    if fault == 'timeout-process':
        (root / 'timed-pid').write_text(str(os.getpid())); os.execl('/bin/sleep', 'sleep', '30')
    print('1 0 init /sbin/init')
    if fault == 'duplicate-process': print('1 0 init /sbin/init')
    if fault == 'diagnostic-process': print('read refused', file=sys.stderr)
    wrapper = str(root / 'state/start-pulse-agent.sh')
    if v['watchdog']:
        if fault == 'script-comm': print('100 1 start-pulse-age /bin/sh ' + wrapper)
        else: print('100 1 sh /bin/sh ' + wrapper if v['platform'] == 'QNAP' else '100 1 bash /bin/bash ' + wrapper)
    if v['collector']: print('101 100 pulse-agent ' + str(root / 'bin/pulse-agent') + ' --token-file /synthetic/private-only')
    if fault == 'foreign-process': print('102 1 pulse-agent /other/instance/pulse-agent --token-file /synthetic/private-only')
    if fault == 'relative-wrapper': print('103 1 sh sh start-pulse-agent.sh')
    if fault == 'deferred-bootstrap': print('104 1 sh /bin/sh /tmp/nasconfig_tmp/autorun.sh')
    if fault == 'live-go': print('105 1 bash /bin/bash ' + str(root / 'go'))
    # rc.local executes its fromdos copy, so the kernel names the shell after it.
    if fault == 'live-tmp-go': print('105 1 go /bin/bash /var/tmp/go')
    if fault == 'live-tmp-go-safemode': print('105 1 go.safemode /bin/bash /var/tmp/go.safemode')
    print('200 1 vi vi ' + wrapper)  # Editor is not a watchdog or signal target.
    print('201 1 sh /bin/sh ' + wrapper + '.bak')
    print('202 1 pulse-agent-prod /other/pulse-agent-prod')
    print('203 1 go /usr/local/go/bin/go build ./...')  # A Go toolchain is not a go shell.
    sys.exit(0)
if role == 'kill':
    assert len(args) == 2 and args[0] == '-TERM', args
    pid = int(args[1]); assert pid in (100, 101), 'unrelated process signalled'
    event('signal:' + str(pid))
    boot = root / ('flash/autorun.sh' if v['platform'] == 'QNAP' else 'go')
    active = boot.exists() and '# Pulse Agent' in boot.read_text()
    event('boot:present-at-signal' if active else 'boot:absent-at-signal')
    if fault == 'refused-watchdog-stop' and pid == 100: sys.exit(1)
    if fault == 'refused-collector-stop' and pid == 101: sys.exit(1)
    if pid == 100 and fault != 'ignored-watchdog-stop': v['watchdog'] = False
    if pid == 101 and fault != 'ignored-collector-stop': v['collector'] = False
    save(); sys.exit(0)
assert role == 'flash' and args[0] in ('mount_flash_config', 'umount_flash_config'), (role, args)
verb = args[0]; event('flash:' + verb)
if fault == 'mount-refusal' and verb == 'mount_flash_config': sys.exit(1)
if fault == 'unmount-refusal' and verb == 'umount_flash_config': sys.exit(1)
sys.exit(0)
'''


def write_model_tools(root, tools):
    # Use the interpreter selected by the native runner, not Darwin's unrelated
    # /usr/bin/python3. These models use only the standard library: repeated
    # site startup is unnecessary, and site hooks must not alter fault controls.
    model = root / 'model.py'
    model.write_text(MODEL)
    for name in ('ps', 'kill', 'flash'):
        tool = tools / name
        tool.write_text('#!/bin/sh\nexec ' + shlex.quote(sys.executable)
                        + ' -I -S ' + shlex.quote(str(model)) + ' ' + name + ' "$@"\n')
        tool.chmod(0o700)


def observed_process_absence(pid_file):
    # /proc does not exist on Darwin. Missing process evidence is not proof
    # of cleanup; only a kernel ESRCH for the recorded positive PID is absence.
    try:
        pid = int(pid_file.read_text())
    except (OSError, ValueError):
        return False
    if pid <= 0:
        return False
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return True
    except OSError:
        return False
    return False


class NASTimingResult(unittest.TextTestResult):
    """Retain progress before the outer deadline, without changing any budget."""

    def startTest(self, test):
        self.started = time.monotonic()
        if not hasattr(self, 'suite_started'):
            self.suite_started = self.started
        self.subcases = 0
        self.progress('start', test)
        super().startTest(test)

    def addSubTest(self, test, subtest, err):
        self.subcases += 1
        super().addSubTest(test, subtest, err)

    def stopTest(self, test):
        super().stopTest(test)
        self.progress('complete', test)

    def progress(self, phase, test):
        now = time.monotonic()
        self.stream.writeln('NAS fixture timing: ' + json.dumps(dict(
            phase=phase, control=test._testMethodName, completed_subcases=self.subcases,
            suite_seconds=round(now - self.suite_started, 6),
            control_seconds=round(now - self.started, 6)), sort_keys=True))
        self.stream.flush()


class AgentNASTeardown(unittest.TestCase):
    """Private model controls; no real machine-wide process inventory."""

    source = SOURCE.read_text()

    def exercise(self, platform, fault="", *, reply="confirmed", boot="canonical", wrapper="regular",
                 state="selected", repeat=False, flow="caller", appliance=True):
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            tools = root / "tools"; tools.mkdir()
            write_model_tools(root, tools)
            for name in ("state", "helper", "profile", "bin", "flash", "legacy", "foreign"):
                d = root / name; d.mkdir(mode=0o700)
                if name in ("state", "helper", "profile", "foreign"):
                    for f in ("runtime.token", "agent-id", "connection.env", "install.sh"):
                        (d / f).write_text("synthetic recovery only\n")
            (root / "legacy/telegraf").write_text("unrelated legacy tool\n")
            target = root / "state/start-pulse-agent.sh"
            other = root / "unrelated"; other.write_text("unchanged unrelated file\n")
            if wrapper == "linked": target.symlink_to(other)
            elif wrapper == "directory": target.mkdir()
            elif wrapper != "missing":
                target.write_text("#!/bin/sh\n# Pulse Agent startup script for " + ("foreign" if wrapper == "foreign" else platform) + "\n")
                target.chmod(0o700)
            vendor = root / "vendor"
            if appliance: vendor.write_text(platform + "\n")
            bootfile = root / "flash/autorun.sh" if platform == "QNAP" else root / "go"
            unrelated = "#!/bin/sh\necho before\nbash /other/start-pulse-agent-prod.sh\necho pulse-agent-prod\n/boot/config/pulse/telegraf/start_telegraf.sh\n"
            bootfile.write_text(unrelated)
            producer = definitions(self.source, ("portable_path_uid", "nas_agent_removal_file_safe", "filter_nas_agent_boot_file", "render_qnap_autorun_block", "append_qnap_autorun_block", "render_unraid_go_launch", "write_unraid_go_entry"))
            # Consume the real production boot emitters, not a second literal.
            if platform == "QNAP":
                command = "AGENT_NAME=pulse-agent BINARY_NAME=pulse-agent INSTALL_DIR=/usr/local/bin\nSTATE_DIR=" + shlex.quote(str(root / "state")) + "\n" + producer + "\nappend_qnap_autorun_block " + shlex.quote(str(bootfile)) + " " + shlex.quote(str(target)) + " " + shlex.quote(str(root / "state"))
            else:
                command = "AGENT_NAME=pulse-agent\n" + producer + "\nwrite_unraid_go_entry " + shlex.quote(str(bootfile)) + " " + shlex.quote(str(target)) + " " + shlex.quote(str(root / "bin/pulse-agent"))
            subprocess.run(["bash", "-c", command], check=True, capture_output=True, text=True)
            bootfile.write_text(bootfile.read_text() + "echo after\n")
            if boot == "duplicate":
                if platform == "QNAP":
                    block = bootfile.read_text().split("# Pulse Agent bootstrap begin", 1)[1].split("# Pulse Agent bootstrap end", 1)[0]
                    bootfile.write_text(bootfile.read_text() + "# Pulse Agent bootstrap begin" + block + "# Pulse Agent bootstrap end\n")
                # Earlier installers' foreground entry still retires with the current one.
                else: bootfile.write_text(bootfile.read_text() + "# Pulse Agent\nbash " + str(target) + "\n")
            elif boot == "foreign": bootfile.write_text(bootfile.read_text().replace(str(target), "/other/start-pulse-agent.sh"))
            elif boot == "indirect": bootfile.write_text(bootfile.read_text() + "bash -c 'bash " + str(target) + "'\n")
            elif boot == "unterminated": bootfile.write_text(bootfile.read_text().replace("# Pulse Agent bootstrap end", "# unknown") if platform == "QNAP" else bootfile.read_text() + "# Pulse Agent\n")
            elif boot == "changed-block": bootfile.write_text(bootfile.read_text().replace("_pulse_wait_max=1800", "_pulse_wait_max=999") if platform == "QNAP" else bootfile.read_text().replace(str(target) + " >>", str(target) + " --unknown >>"))
            elif boot == "no-newline": bootfile.write_text(bootfile.read_text().rstrip("\n"))
            elif boot == "binary": bootfile.write_bytes(bootfile.read_bytes() + b'\0\n')
            elif boot == "oversized": bootfile.write_text(bootfile.read_text() + "#" * 1048577 + "\n")
            elif boot == "absent": bootfile.write_text(unrelated + "echo after\n")
            elif boot == "missing": bootfile.unlink()
            elif boot in ("linked", "directory"):
                bootfile.unlink()
                if boot == "linked": bootfile.symlink_to(other)
                else: bootfile.mkdir()
            original_boot = bootfile.read_bytes() if bootfile.is_file() and not bootfile.is_symlink() else None
            if fault == "prior-recovery": (Path(str(bootfile) + ".pulse-removal.retained")).mkdir()
            if fault == "missing-flash": (tools / "flash").unlink()
            model = dict(platform=platform, fault=fault, reads=0, watchdog=wrapper != "missing", collector=wrapper != "missing")
            if boot in ("absent", "missing") and wrapper == "missing": model.update(watchdog=False, collector=False)
            (root / "model.json").write_text(json.dumps(model))
            before = self.snapshot(root)

            replacements = {"/etc/unraid-version": str(vendor if platform == "Unraid" else root / "absent-vendor"),
                            "/boot/config/plugins/pulse-agent": str(root / "absent-plugin"),
                            "/etc/config/qpkg.conf": str(vendor if platform == "QNAP" else root / "absent-vendor"),
                            "/sbin/getcfg": str(root / "absent-getcfg"),
                            "/etc/init.d/init_disk.sh": str(tools / "flash"),
                            "/tmp/nasconfig_tmp": str(root / "flash"),
                            "/boot/config/go": str(root / "go"),
                            "[[ -d /boot/config && ! -L /boot/config ]]": "[[ -d " + str(root) + " && ! -L " + str(root) + " ]]",
                            "/boot/config/pulse": str(root / "legacy"),
                            "[[ -d /etc/init.d ]]": "[[ -d " + str(root / "no-init") + " ]]",
                            "/usr/syno": str(root / "no-dsm"),
                            "/var/log/pulse-agent.log": str(root / "absent-log"),
                            "/var/log/pulse": str(root / "absent-log-dir"),
                            "/etc/truenas-version": str(root / "no-truenas"),
                            "/etc/version": str(root / "no-version")}
            def redirect(text):
                for a, b in replacements.items(): text = text.replace(a, b)
                return text
            values = dict(FIXTURE=root, EVENTS=root / "events", PLATFORM=platform, FAULT=fault,
                          STATE_DIR=root / "state", STATE_DIR_REMOVAL_AUTHORITY=(root / "state" if state == "selected" else root / "foreign"),
                          INSTALL_DIR=root / "bin", PRIVILEGED_HELPER_STATE_DIR=root / "helper",
                          SAFE_PROFILE_STATE_DIR=root / "profile", PRIVILEGE_SUDOERS_FILE=root / "absent-sudoers",
                          PRIVILEGE_HELPER_DIR=root / "absent-helper", TRUENAS_STATE_DIR=root / "no-truenas-state",
                          TRUENAS_BOOTSTRAP_SCRIPT=root / "no-bootstrap", BOOT_PATH=bootfile, REPLY=reply)
            setup = "\n".join(k + "=" + shlex.quote(str(v)) for k, v in values.items())
            shell = r'''
set -euo pipefail
UNINSTALL=true AGENT_NAME=pulse-agent BINARY_NAME=pulse-agent
EXIT_GENERAL=1 EXIT_MISSING_ARGS=16 PULSE_URL=https://same-instance.example.invalid AGENT_ID=synthetic
ACTION_RUNNER_NAME=pulse-agent-runner PRIVILEGED_HELPER_NAME=pulse-agent-helper
log_info() { :; }; log_warn() { :; }
fail() { printf '%s\n' "$1" >&2; exit "$2"; }
uname() { printf 'Linux\n'; }
command() {
    case "$*" in
        '-v systemctl'|'-v rc-service'|'-v initctl') return 1 ;;
        '-v timeout') [[ "$FAULT" != missing-timeout ]] || return 1 ;;
        '-v ps') [[ "$FAULT" != missing-ps ]] || return 1 ;;
    esac
    builtin command "$@"
}
truenas_scale_agent_removal_present() { return 1; }
collector_credential_state_present() { [[ -e "$STATE_DIR/runtime.token" ]]; }
uninstall_collector_registration() {
    printf 'remote:remove\n' >> "$EVENTS"
    python3 -c 'import json,os,pathlib; r=pathlib.Path(os.environ["FIXTURE"]); v=json.loads((r/"model.json").read_text()); b=r/("flash/autorun.sh" if v["platform"]=="QNAP" else "go"); e=[]; e += ["remote:processes-active"] if v["watchdog"] or v["collector"] else []; e += ["remote:boot-present"] if b.exists() and "# Pulse Agent" in b.read_text() else []; f=(r/"events").open("a"); f.write("\n".join(e)+("\n" if e else "")); f.close()'
    if [[ "$FAULT" == late-boot ]]; then
        cp "$NAS_REMOVAL_BOOT_RECOVERY/original" "$BOOT_PATH"
    elif [[ "$FAULT" == late-process ]]; then
        python3 -c 'import json,os,pathlib; p=pathlib.Path(os.environ["FIXTURE"])/"model.json"; v=json.loads(p.read_text()); v["watchdog"]=True; p.write_text(json.dumps(v))'
    fi
    [[ "$REPLY" == confirmed ]]
}
find_qnap_state_dir() { [[ "$PLATFORM" != QNAP ]] || printf '%s\n' "$STATE_DIR"; }
pkill() { printf 'unchecked:process-stop\n' >> "$EVENTS"; }
kill() { builtin command "$FIXTURE/tools/kill" "$@"; }
sleep() { :; }
rm() {
    local arg
    for arg in "$@"; do [[ "$arg" == -* || "$arg" == "$FIXTURE/"* ]] || return 97; done
    printf 'file:remove\n' >> "$EVENTS"; builtin command rm "$@"
}
cp() { [[ "$FAULT" != refused-copy ]] || return 1; builtin command cp "$@"; }
chmod() { [[ "$FAULT" != refused-chmod || "${@: -1}" != "$BOOT_PATH" ]] || return 1; builtin command chmod "$@"; }
mv() {
    [[ "$FAULT" != refused-edit ]] || return 1
    if [[ "$FAULT" == source-changed ]]; then printf 'echo concurrent-owner\n' >> "$1"; fi
    builtin command mv "$@" || return 1
    case "$FAULT" in
        concurrent-file) printf 'concurrent boot owner\n' > "$BOOT_PATH" ;;
        concurrent-directory) mkdir "$BOOT_PATH" ;;
        concurrent-symlink) ln -s "$FIXTURE/unrelated" "$BOOT_PATH" ;;
    esac
}
remove_agent_state_dir() { printf 'state:remove\n' >> "$EVENTS"; rm -rf -- "$STATE_DIR"; }
remove_privileged_helper_state_dir() { rm -rf -- "$PRIVILEGED_HELPER_STATE_DIR"; }
remove_safe_profile_state_dir() { rm -rf -- "$SAFE_PROFILE_STATE_DIR"; }
'''
            entry = self.source.split("# --- Uninstall Logic ---\n", 1)[1].split("# --- Validation ---\n", 1)[0]
            if flow == "process": entry = 'read_nas_agent_removal_processes "$PLATFORM"\n'
            executable = shell + "\n" + setup + "\n" + redirect(definitions(self.source)) + "\nmain() {\n" + redirect(entry) + "}\nmain\n"
            env = dict(os.environ, FIXTURE=str(root), PATH=str(tools) + ":" + os.environ["PATH"])
            result = subprocess.run(["bash", "-c", executable], env=env, capture_output=True, text=True, timeout=14)
            second = subprocess.run(["bash", "-c", executable], env=env, capture_output=True, text=True, timeout=14) if repeat else None
            events = (root / "events").read_text().splitlines() if (root / "events").exists() else []
            after = self.snapshot(root)
            content = bootfile.read_bytes() if bootfile.is_file() and not bootfile.is_symlink() else None
            recovered = {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("retired")}
            timed_stopped = observed_process_absence(root / "timed-pid")
            return result, second, events, before, after, json.loads((root / "model.json").read_text()), content, original_boot, recovered, timed_stopped

    @staticmethod
    def snapshot(root):
        result = {}
        for p in root.rglob("*"):
            if p.parts[len(root.parts)] not in ("state", "helper", "profile", "unrelated", "legacy", "foreign"):
                continue
            mode = p.lstat().st_mode
            data = os.readlink(p) if stat.S_ISLNK(mode) else p.read_bytes() if stat.S_ISREG(mode) else None
            result[str(p.relative_to(root))] = (mode, data, p.lstat().st_ino)
        return result

    def retained(self, result, *, remote=False):
        execution, _, events, before, after, _, _, _, _, _ = result
        self.assertEqual(execution.returncode, 1, execution.stdout + execution.stderr)
        self.assertNotIn('state:remove', events)
        self.assertEqual('remote:remove' in events, remote)
        for path in before: self.assertEqual(after.get(path), before[path], path)
        self.assertNotIn('synthetic recovery only', execution.stdout + execution.stderr)
        self.assertNotIn('/synthetic/private-only', execution.stdout + execution.stderr)
        return events

    def test_complete_process_evidence_is_required_before_boot_or_remote_mutation(self):
        for platform in ('Unraid', 'QNAP'):
            for fault in ('failed-process', 'empty-process', 'duplicate-process', 'diagnostic-process', 'missing-timeout', 'missing-ps'):
                with self.subTest(platform=platform, fault=fault):
                    result = self.exercise(platform, fault); events = self.retained(result)
                    self.assertEqual(result[6], result[7]); self.assertFalse(any(e.startswith('signal:') for e in events))

    def test_unknown_relative_foreign_or_deferred_boot_owners_are_not_signalled(self):
        for platform, fault in (('Unraid', 'foreign-process'), ('QNAP', 'foreign-process'), ('Unraid', 'relative-wrapper'), ('QNAP', 'deferred-bootstrap')):
            with self.subTest(platform=platform, fault=fault):
                result = self.exercise(platform, fault); events = self.retained(result)
                self.assertEqual(result[6], result[7]); self.assertFalse(any(e.startswith('signal:') for e in events))

    def test_non_regular_foreign_wrapper_and_unselected_state_preserve_recovery(self):
        for platform in ('Unraid', 'QNAP'):
            for wrapper in ('linked', 'directory', 'foreign'):
                with self.subTest(platform=platform, wrapper=wrapper):
                    result = self.exercise(platform, wrapper=wrapper); self.retained(result); self.assertEqual(result[6], result[7])
            self.retained(self.exercise(platform, state='foreign'))

    def test_only_complete_owned_boot_entries_are_admitted(self):
        for platform in ('Unraid', 'QNAP'):
            for boot in ('foreign', 'indirect', 'unterminated', 'changed-block', 'no-newline', 'binary', 'oversized', 'linked', 'directory'):
                with self.subTest(platform=platform, boot=boot):
                    result = self.exercise(platform, boot=boot); self.retained(result); self.assertEqual(result[6], result[7])

    def test_refused_edit_copy_or_prior_recovery_cannot_erase_credentials(self):
        for platform in ('Unraid', 'QNAP'):
            for fault in ('refused-edit', 'refused-copy', 'refused-chmod', 'prior-recovery'):
                with self.subTest(platform=platform, fault=fault): self.retained(self.exercise(platform, fault))

    def test_concurrent_boot_file_directory_and_symlink_are_not_clobbered(self):
        for platform in ('Unraid', 'QNAP'):
            for fault in ('concurrent-file', 'concurrent-directory', 'concurrent-symlink', 'source-changed'):
                with self.subTest(platform=platform, fault=fault):
                    result = self.exercise(platform, fault); self.retained(result)
                    self.assertTrue(result[8], 'actual retired boot recovery must survive')
                    if fault == 'concurrent-file': self.assertEqual(result[6], b'concurrent boot owner\n')
                    if fault == 'source-changed': self.assertTrue(any(b'echo concurrent-owner\n' in data for data in result[8].values()))

    def test_flash_mount_and_unmount_failures_are_not_local_only_permission(self):
        for fault in ('mount-refusal', 'unmount-refusal', 'missing-flash'):
            with self.subTest(fault=fault):
                events = self.retained(self.exercise('QNAP', fault)); self.assertFalse(any(e.startswith('signal:') for e in events))
                if fault != 'missing-flash': self.assertIn('flash:umount_flash_config', events)

    def test_watchdog_stops_before_collector_and_ignored_or_refused_signals_retain_state(self):
        for platform in ('Unraid', 'QNAP'):
            for fault in ('ignored-watchdog-stop', 'refused-watchdog-stop', 'ignored-collector-stop', 'refused-collector-stop'):
                with self.subTest(platform=platform, fault=fault):
                    events = self.retained(self.exercise(platform, fault))
                    self.assertNotIn('unchecked:process-stop', events)
                    if 'watchdog' in fault: self.assertNotIn('signal:101', events)

    def test_all_owned_entries_disable_before_signalling_and_remote_erasure(self):
        for platform, fault in (('Unraid', ''), ('QNAP', ''), ('QNAP', 'script-comm')):
            with self.subTest(platform=platform, fault=fault):
                r = self.exercise(platform, fault, boot='duplicate')
                self.assertEqual(r[0].returncode, 0, r[0].stderr)
                self.assertLess(r[2].index('signal:100'), r[2].index('signal:101'))
                self.assertLess(r[2].index('signal:101'), r[2].index('remote:remove'))
                self.assertLess(r[2].index('remote:remove'), r[2].index('state:remove'))
                self.assertNotIn('unchecked:process-stop', r[2]); self.assertFalse(r[5]['watchdog'] or r[5]['collector'])
                self.assertEqual(r[2].count('boot:absent-at-signal'), 2)
                for illegal in ('boot:present-at-signal', 'remote:processes-active', 'remote:boot-present'):
                    self.assertNotIn(illegal, r[2])
                for key in ('unrelated', 'legacy', 'legacy/telegraf'): self.assertEqual(r[3][key], r[4][key])
                self.assertIn(b'bash /other/start-pulse-agent-prod.sh\n', r[6]); self.assertIn(b'echo after\n', r[6])
                self.assertNotIn(b'# Pulse Agent\n', r[6]); self.assertNotIn(b'# Pulse Agent bootstrap', r[6]); self.assertEqual(r[8], {})

    def test_remote_refusal_keeps_disabled_boot_and_exact_local_recovery(self):
        for platform in ('Unraid', 'QNAP'):
            with self.subTest(platform=platform):
                r = self.exercise(platform, reply='refused'); self.retained(r, remote=True)
                self.assertFalse(r[5]['watchdog'] or r[5]['collector']); self.assertNotIn(b'# Pulse Agent', r[6]); self.assertTrue(r[8])

    def test_reactivation_before_remote_refuses_without_erasure(self):
        for platform in ('Unraid', 'QNAP'):
            with self.subTest(platform=platform): self.retained(self.exercise(platform, 'reactivated-before-remote'))

    def test_late_boot_or_process_activation_retains_recovery_after_remote(self):
        for platform in ('Unraid', 'QNAP'):
            for fault in ('late-boot', 'late-process'):
                with self.subTest(platform=platform, fault=fault): self.retained(self.exercise(platform, fault), remote=True)

    def test_absent_missing_and_repeated_removal_require_fresh_quiescence(self):
        for platform in ('Unraid', 'QNAP'):
            for boot in ('absent', 'missing'):
                with self.subTest(platform=platform, boot=boot):
                    r = self.exercise(platform, boot=boot, wrapper='missing', repeat=True)
                    self.assertEqual(r[0].returncode, 0, r[0].stderr); self.assertEqual(r[1].returncode, 0, r[1].stderr)
                    self.assertEqual(r[2].count('remote:remove'), 1); self.assertNotIn('unchecked:process-stop', r[2])
                    self.assertGreater(r[2].count('process:read'), 4)

    def test_process_observation_returns_roles_not_arguments_and_stops_timeout_child(self):
        r = self.exercise('Unraid', flow='process')
        self.assertEqual(r[0].returncode, 0, r[0].stderr)
        self.assertIn('100 1 watchdog\n', r[0].stdout); self.assertIn('101 100 collector\n', r[0].stdout)
        self.assertNotIn('/synthetic/private-only', r[0].stdout); self.assertIn('200 1 other\n', r[0].stdout)
        self.assertIn('203 1 other\n', r[0].stdout)
        r = self.exercise('QNAP', 'timeout-process'); self.retained(r); self.assertTrue(r[9])

    def test_private_model_uses_selected_isolated_interpreter(self):
        with tempfile.TemporaryDirectory(prefix='nas model ') as directory:
            root = Path(directory)
            tools = root / 'tools'; tools.mkdir()
            invocations = root / 'interpreter-arguments'
            interpreter = root / 'selected python'
            interpreter.write_text('#!/bin/sh\nprintf "%s\\n" "$@" >> '
                                   + shlex.quote(str(invocations)) + '\nexec '
                                   + shlex.quote(sys.executable) + ' "$@"\n')
            interpreter.chmod(0o700)
            with mock.patch.object(sys, 'executable', str(interpreter)):
                write_model_tools(root, tools)
            # A site hook would contaminate model state if startup isolation
            # disappeared. No external package, service or process is involved.
            (root / 'sitecustomize.py').write_text(
                'from pathlib import Path\nPath(' + repr(str(root / 'site-ran')) + ').touch()\n')
            (root / 'model.json').write_text(json.dumps(dict(
                platform='QNAP', fault='', reads=0, watchdog=True, collector=True)))
            (root / 'flash').mkdir(); (root / 'flash/autorun.sh').write_text('#!/bin/sh\n')
            env = dict(os.environ, FIXTURE=str(root), PYTHONPATH=str(root))
            commands = (
                ('ps', '-ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'comm=', '-o', 'args='),
                ('kill', '-TERM', '100'),
                ('flash', 'mount_flash_config'),
            )
            for name, *args in commands:
                result = subprocess.run([str(tools / name), *args], env=env,
                                        capture_output=True, text=True, timeout=5)
                self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
                self.assertEqual(result.stderr, '')
            recorded = invocations.read_text().splitlines()
            self.assertEqual(recorded.count('-I'), 3)
            self.assertEqual(recorded.count('-S'), 3)
            self.assertEqual(recorded.count(str(root / 'model.py')), 3)
            self.assertFalse((root / 'site-ran').exists())
            self.assertEqual((root / 'events').read_text().splitlines(), [
                'process:read', 'signal:100', 'boot:absent-at-signal', 'flash:mount_flash_config'])
            self.assertFalse(json.loads((root / 'model.json').read_text())['watchdog'])

    def test_timeout_child_absence_requires_observed_exit(self):
        with tempfile.TemporaryDirectory() as directory:
            pid_file = Path(directory) / 'child-pid'
            with mock.patch.object(os, 'kill') as probe:
                for value in (None, '', 'invalid', '0', '-1'):
                    if value is not None:
                        pid_file.write_text(value)
                    self.assertFalse(observed_process_absence(pid_file))
                probe.assert_not_called()
            process = subprocess.Popen(['sleep', '30'])
            try:
                pid_file.write_text(str(process.pid))
                self.assertFalse(observed_process_absence(pid_file), 'live child reported absent')
                with mock.patch.object(os, 'kill', side_effect=PermissionError):
                    self.assertFalse(observed_process_absence(pid_file), 'refusal reported absence')
            finally:
                process.terminate()
                process.wait(timeout=3)
            self.assertTrue(observed_process_absence(pid_file), 'reaped child still reported live')

    def test_control_timing_reports_cumulative_and_per_control_time(self):
        import io

        class InertControl(unittest.TestCase):
            def test_inert(self):
                for _ in range(2):
                    with self.subTest():
                        self.assertTrue(True)

        stream = io.StringIO()
        result = unittest.TextTestRunner(stream=stream, verbosity=2,
                                         resultclass=NASTimingResult).run(
                                             unittest.defaultTestLoader.loadTestsFromTestCase(InertControl))
        self.assertTrue(result.wasSuccessful())
        records = [json.loads(line.removeprefix('NAS fixture timing: '))
                   for line in stream.getvalue().splitlines() if line.startswith('NAS fixture timing: ')]
        self.assertEqual([r['phase'] for r in records], ['start', 'complete'])
        self.assertEqual([r['completed_subcases'] for r in records], [0, 2])
        for record in records:
            self.assertEqual(set(record), {'phase', 'control', 'completed_subcases',
                                           'suite_seconds', 'control_seconds'})
            self.assertEqual(record['control'], 'test_inert')
            self.assertGreaterEqual(record['suite_seconds'], record['control_seconds'])
        self.assertGreaterEqual(records[1]['control_seconds'], records[0]['control_seconds'])

    def test_live_unraid_boot_shell_is_unknown_not_a_signal_target(self):
        for fault in ('live-go', 'live-tmp-go', 'live-tmp-go-safemode'):
            with self.subTest(fault=fault):
                r = self.exercise('Unraid', fault); events = self.retained(r)
                self.assertEqual(r[6], r[7]); self.assertFalse(any(e.startswith('signal:') for e in events))

    def test_recovered_ordinary_linux_state_is_not_qnap_platform_authority(self):
        r = self.exercise('QNAP', appliance=False)
        self.assertEqual(r[0].returncode, 0, r[0].stderr)
        self.assertFalse(any(e.startswith('flash:') or e.startswith('signal:') for e in r[2]))
        for key in ('foreign', 'foreign/runtime.token', 'legacy/telegraf'):
            self.assertEqual(r[3][key], r[4][key])


    def test_private_process_reader_requests_native_basename_and_full_argv(self):
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            native = root / 'native ps'
            # Fixed synthetic argv recorder, not a process-inventory model.
            native.write_text('#!/bin/sh\nprintf "%s\\n" "$@"\n')
            native.chmod(0o700)
            tools = root / 'tools'; tools.mkdir()
            adapter = write_wide_ps_adapter(tools, native)
            for fields in (
                ['-ax', '-o', 'comm=', '-o', 'args='],
                ['-ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'comm=', '-o', 'args='],
            ):
                with self.subTest(fields=fields):
                    result = subprocess.run([str(adapter), *fields], capture_output=True, text=True, check=True)
                    self.assertEqual(result.stdout.splitlines(),
                                     ['-ww', *(field if field != 'comm=' else 'ucomm=' for field in fields)])

    def test_generated_collector_observation_keeps_exact_identity_on_both_unix_formats(self):
        runtime = Path('/synthetic/owned/bin/pulse-agent')
        for comm in ('pulse-agent', str(runtime)):
            with self.subTest(comm=comm):
                self.assertTrue(generated_collector_is_running(f'{comm} {runtime} 30\n', runtime))
        for listing in (
            f'other {runtime} 30\n',
            'pulse-agent /synthetic/foreign/bin/pulse-agent 30\n',
            f'pulse-agent {runtime}-prod 30\n',
            f'pulse-agent /bin/sh echo {runtime} 30\n',
            f'/synthetic/owne {runtime} 30\n',  # Original Darwin middle-column truncation.
            f'pulse-agent {runtime}\n',
            'pulse-agent\n',
            '',
        ):
            with self.subTest(listing=listing):
                self.assertFalse(generated_collector_is_running(listing, runtime))

    def test_caller_has_no_blanket_nas_or_legacy_tree_deletion(self):
        caller = self.source.split('# --- Uninstall Logic ---\n', 1)[1].split('# --- Validation ---\n', 1)[0]
        self.assertNotIn("sed -i '/pulse-agent/d'", caller)
        self.assertNotIn('rm -rf /boot/config/pulse', caller)
        self.assertNotIn('rm -rf "$qnap_state_dir"', caller)
        self.assertIn('teardown_nas_agent_boot_owner "$uninstall_nas_platform"', caller)
        self.assertIn('[[ "$uninstall_upstart" != true && -z "$uninstall_nas_platform" ]]', caller)


class AgentNASLiveTeardown(unittest.TestCase):
    """Real machine-wide process observation; the Go caller keeps this serial."""

    source = SOURCE.read_text()

    def test_actual_generated_watchdogs_and_children_become_quiescent(self):
        # Private ordinary-user processes only. The full Go suite runs this in
        # the standard offline source VM; it is not native NAS/root acceptance.
        import signal
        import time
        for platform in ('Unraid', 'QNAP'):
            with self.subTest(platform=platform), tempfile.TemporaryDirectory() as directory:
                # Darwin's temporary-directory alias must not become a
                # different executable identity in its process observation.
                root = Path(directory).resolve()
                for name in ('state', 'bin', 'tools', 'flash', 'logs'): (root / name).mkdir(mode=0o700)
                runtime = root / 'bin/pulse-agent'; stored = root / 'state/stored-agent'
                # Supply a native Go executable like the actual collector,
                # not a renamed platform-owned /bin/sleep with implicit
                # executable/process semantics. The generated production
                # wrapper still copies it to the exact runtime path, and real
                # kernel rows still drive the unchanged ownership/stop checks.
                build = subprocess.run(
                    ['go', 'build', '-trimpath', '-o', str(stored),
                     './scripts/installtests/nasfixturecollector'],
                    cwd=ROOT, capture_output=True, text=True, timeout=30,
                )
                self.assertEqual(build.returncode, 0, build.stdout + build.stderr)
                self.assertTrue(stored.is_file())
                self.assertFalse(stored.is_symlink())
                stored.chmod(0o700)
                watchdog = root / 'state/start-pulse-agent.sh'
                (root / 'tools/pkill').write_text('#!/bin/sh\nexit 0\n'); (root / 'tools/pkill').chmod(0o700)
                (root / 'tools/flash').write_text('#!/bin/sh\nexit 0\n'); (root / 'tools/flash').chmod(0o700)
                # Capture the native executable before placing the adapter on
                # PATH. The same untruncated kernel rows feed startup and the
                # unchanged production teardown parser (no model or PID claim).
                native_ps = shutil.which('ps')
                self.assertIsNotNone(native_ps, 'native process reader missing')
                fixture_ps = write_wide_ps_adapter(root / 'tools', Path(native_ps).resolve())
                env = dict(os.environ, PATH=str(root / 'tools') + ':' + os.environ['PATH'])
                setup = '\n'.join(k + '=' + shlex.quote(str(v)) for k, v in dict(
                    STATE_DIR=root / 'state', STATE_DIR_REMOVAL_AUTHORITY=root / 'state', INSTALL_DIR=root / 'bin',
                    WRAPPER_SCRIPT=watchdog, UNRAID_STORED_BINARY=stored, RUNTIME_BINARY=runtime,
                    AGENT_LOG_FILE=root / 'logs/agent.log').items())
                setup += '\nAGENT_NAME=pulse-agent BINARY_NAME=pulse-agent EXEC_ARGS=30\nEXPORT_SERVICE_ENV="" SHELL_EXPORT_LINES=""\n'
                if platform == 'QNAP':
                    renderer = definitions(self.source, ('write_qnap_wrapper_script',))
                    render = setup + renderer + '\nwrite_qnap_wrapper_script "$WRAPPER_SCRIPT" "$RUNTIME_BINARY" "$UNRAID_STORED_BINARY" ' + shlex.quote(str(root / 'logs')) + ' "$STATE_DIR"\n'
                else:
                    start = self.source.index('    cat > "$WRAPPER_SCRIPT" <<EOF', self.source.index('# 3. Unraid'))
                    end = self.source.index('\nEOF\n', start) + 5
                    render = setup + self.source[start:end].replace('/var/log/${AGENT_NAME}-watchdog.log', str(root / 'logs/watchdog.log'))
                render_file = root / 'render.sh'; render_file.write_text(render)
                rendered = subprocess.run(['bash', str(render_file)], env=env, capture_output=True, text=True)
                self.assertEqual(rendered.returncode, 0, rendered.stdout + rendered.stderr)
                bootfile = root / 'flash/autorun.sh' if platform == 'QNAP' else root / 'go'
                if platform == 'QNAP':
                    bootfile.write_text('#!/bin/sh\n')
                    emitter = definitions(self.source, ('portable_path_uid', 'nas_agent_removal_file_safe', 'filter_nas_agent_boot_file', 'render_qnap_autorun_block', 'append_qnap_autorun_block'))
                    render_file.write_text(setup + emitter + '\nappend_qnap_autorun_block ' + shlex.quote(str(bootfile)) + ' "$WRAPPER_SCRIPT" "$STATE_DIR"\n')
                    rendered = subprocess.run(['bash', str(render_file)], env=env, capture_output=True, text=True)
                    self.assertEqual(rendered.returncode, 0, rendered.stdout + rendered.stderr)
                else:
                    emitter = definitions(self.source, ('portable_path_uid', 'nas_agent_removal_file_safe', 'filter_nas_agent_boot_file', 'render_unraid_go_launch', 'write_unraid_go_entry'))
                    render_file.write_text(setup + emitter + '\nwrite_unraid_go_entry ' + shlex.quote(str(bootfile)) + ' "$WRAPPER_SCRIPT" "$RUNTIME_BINARY"\n')
                    rendered = subprocess.run(['bash', str(render_file)], env=env, capture_output=True, text=True)
                    self.assertEqual(rendered.returncode, 0, rendered.stdout + rendered.stderr)
                functions = definitions(self.source)
                functions = functions.replace('/boot/config/go', str(bootfile))
                functions = functions.replace('[[ -d /boot/config && ! -L /boot/config ]]', '[[ -d ' + str(root) + ' && ! -L ' + str(root) + ' ]]')
                functions = functions.replace('/etc/init.d/init_disk.sh', str(root / 'tools/flash'))
                functions = functions.replace('/tmp/nasconfig_tmp', str(root / 'flash'))
                control = root / 'control.sh'
                control.write_text('set -euo pipefail\n' + setup + '\n' + functions + '\ncommand() { [[ "$*" != "-v systemctl" ]] || return 1; builtin command "$@"; }\nquiesce_nas_agent_for_removal ' + platform + '\n')
                output = root / 'watchdog.log'
                collector_log = root / 'logs' / ('pulse-agent-watchdog.log' if platform == 'QNAP' else 'watchdog.log')
                with output.open('wb') as stream:
                    process = subprocess.Popen([str(watchdog)] if platform == 'QNAP' else ['bash', str(watchdog)], env=env,
                                               stdout=stream, stderr=stream, start_new_session=True)
                    try:
                        deadline = time.monotonic() + 8
                        while time.monotonic() < deadline:
                            listing = subprocess.run([str(fixture_ps), '-ax', '-o', 'comm=', '-o', 'args='], capture_output=True, text=True, check=True).stdout
                            if (generated_collector_is_running(listing, runtime)
                                    and collector_log.is_file()
                                    and 'private NAS fixture collector ready\n' in collector_log.read_text()): break
                            self.assertIsNone(process.poll(), output.read_text()); time.sleep(.05)
                        else:
                            # Retain only owned fixture rows/logs, never other
                            # process arguments, if hosted startup still fails.
                            owned = '\n'.join(line for line in listing.splitlines() if str(root) in line)
                            logs = '\n'.join(p.read_text() for p in (root / 'logs').glob('*.log'))
                            self.fail('private generated collector did not start\n' + owned + '\n' + output.read_text() + '\n' + logs)
                        result = subprocess.run(['bash', str(control)], env=env, capture_output=True, text=True, timeout=12)
                        self.assertEqual(result.returncode, 0, result.stdout + result.stderr + output.read_text())
                        process.wait(timeout=3)
                        with self.assertRaises(ProcessLookupError): os.killpg(process.pid, 0)
                        self.assertNotIn('# Pulse Agent', bootfile.read_text())
                    finally:
                        try: os.killpg(process.pid, signal.SIGKILL)
                        except ProcessLookupError: pass
                        process.wait(timeout=3)


if __name__ == '__main__':
    unittest.main(verbosity=2, testRunner=unittest.TextTestRunner(verbosity=2, resultclass=NASTimingResult))