Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

82.3 KB
Raw Blame
from __future__ import annotations

import json
import os
import subprocess
import sys
import threading
import time
from pathlib import Path

import psutil
import pytest

from config.v2_config import V2Config
from tests import process_reaper
from tests.test_remote_access_vibe_cloud import _config
from vibe import remote_access, runtime

# Real fake-cloudflared connectors and log sinks, some started by child Pythons.
pytestmark = [pytest.mark.reap_processes, pytest.mark.usefixtures("reap_helper_spawns")]


def _quality(median: float) -> dict:
    return {
        "ha_connections": 4,
        "rtt_ms": {"min": median - 20, "median": median, "max": median + 40},
        "request_errors_per_minute": 0,
        "packet_loss_per_minute": 0,
    }


def _request_path(
    p50: float,
    p95: float,
    p99: float,
    *,
    slow_rate: float = 0.0,
    failure_rate: float = 0.0,
) -> dict:
    return {
        "source": "synthetic_local",
        "status": "degraded" if p95 >= 750 or slow_rate >= 0.05 else "healthy",
        "confidence": "high",
        "window_seconds": 180,
        "sample_count": 100,
        "success_count": round(100 * (1 - failure_rate)),
        "latency_ms": {"p50": p50, "p95": p95, "p99": p99, "max": p99},
        "failure_rate": failure_rate,
        "slow_request_rate": {
            "over_500_ms": slow_rate,
            "over_1000_ms": slow_rate,
            "over_2000_ms": 0.0,
        },
        "baseline_p95_ms": 250.0,
    }


def _metrics_sample(*, connections: int = 4) -> remote_access.tunnel_quality.MetricsSample:
    return remote_access.tunnel_quality.MetricsSample(
        sampled_at=time.time(),
        ready=connections > 0,
        ha_connections=connections,
        edge_locations=("sin01", "sin02"),
        smoothed_rtt_ms=(70, 75, 80, 85),
        request_errors_total=0,
        packet_loss_total=0,
        closed_connections_total=0,
    )


def _setup_recovery(monkeypatch, tmp_path, candidate_quality: dict):
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.save()
    binary = "/usr/local/bin/cloudflared"
    active_pid = 111
    candidate_pid = 222
    alive = {active_pid, candidate_pid}
    remote_access._RECOVERY_CANCEL_EVENT.clear()
    remote_access._pid_path().parent.mkdir(parents=True, exist_ok=True)
    remote_access._pid_path().write_text(str(active_pid), encoding="utf-8")
    remote_access._write_state(active_pid, config, binary, "http://127.0.0.1:29001")

    monkeypatch.setattr(remote_access, "tunnel_quality_snapshot", lambda: _quality(250))
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)
    monkeypatch.setattr(remote_access, "_report_runtime_status_async", lambda *args, **kwargs: None)
    monkeypatch.setattr(remote_access, "_resolve_binary", lambda loaded: binary)
    monkeypatch.setattr(remote_access, "_allocate_metrics_url", lambda: "http://127.0.0.1:29002")
    monkeypatch.setattr(remote_access, "_wait_candidate_ready", lambda pid, url: True)
    monkeypatch.setattr(remote_access, "_candidate_average_snapshot", lambda url: candidate_quality)
    monkeypatch.setattr(remote_access.tunnel_quality, "scrape_metrics", lambda url: _metrics_sample())
    monkeypatch.setattr(runtime, "pid_alive", lambda pid: pid in alive)
    monkeypatch.setattr(runtime, "get_process_command", lambda pid: f"{binary} tunnel run")

    class ReadyResponse:
        ok = True

    monkeypatch.setattr(remote_access.requests, "get", lambda *args, **kwargs: ReadyResponse())

    def spawn_background(args, pid_path, stdout_name, stderr_name, env=None):
        pid_path.write_text(str(candidate_pid), encoding="utf-8")
        return candidate_pid

    monkeypatch.setattr(runtime, "spawn_background", spawn_background)
    return config, active_pid, candidate_pid, alive


def test_ra_tq_004_candidate_promotes_before_active_drains(monkeypatch, tmp_path) -> None:
    """RA-TQ-004: Better candidate is promoted before the active Connector drains."""

    config, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    drained = []
    results = []

    def stop_pid(pid, timeout=8):
        drained.append((pid, timeout, remote_access._read_pid()))
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    remote_access._run_route_optimization(config, "latency")

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == candidate_pid
    assert state["active"]["pid"] == candidate_pid
    assert state["candidate"] is None
    assert state["draining"] is None
    assert drained == [(active_pid, remote_access.RECOVERY_DRAIN_SECONDS, candidate_pid)]
    assert results[0]["result"] == "improved"


def test_ra_tq_005_non_improving_candidate_keeps_active(monkeypatch, tmp_path) -> None:
    """RA-TQ-005: Non-improving candidate is removed without changing the active Connector."""

    config, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(220))
    stopped = []
    results = []

    def stop_pid(pid, timeout=8):
        stopped.append(pid)
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    remote_access._run_route_optimization(config, "latency")

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == active_pid
    assert state["active"]["pid"] == active_pid
    assert state["candidate"] is None
    assert stopped == [candidate_pid]
    assert results[0]["result"] == "no_improvement"


def test_ra_tq_028_settings_candidate_failure_keeps_active_config(monkeypatch, tmp_path) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.save()
    events = []
    monkeypatch.setattr(remote_access, "status", lambda loaded=None: {"running": True})
    monkeypatch.setattr(remote_access, "_resolve_binary", lambda loaded: "/usr/bin/cloudflared")
    monkeypatch.setattr(
        remote_access,
        "_start_candidate_connector",
        lambda loaded, binary, protocol: events.append(("start", protocol)) or (222, "http://metrics"),
    )
    monkeypatch.setattr(
        remote_access,
        "_wait_candidate_ready",
        lambda pid, metrics: events.append(("ready", pid)) or False,
    )
    monkeypatch.setattr(
        remote_access,
        "_discard_candidate_connector",
        lambda pid: events.append(("discard", pid)),
    )
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = None

    result = remote_access.apply_settings({"transport_protocol": "http2"})

    assert result["ok"] is False
    assert result["error"] == "remote_access_settings_apply_failed"
    assert V2Config.load().remote_access.vibe_cloud.transport_protocol == "auto"
    assert events == [("start", "http2"), ("ready", 222), ("discard", 222)]


def test_ra_tq_028_settings_promote_before_draining_previous_connector(monkeypatch, tmp_path) -> None:
    """RA-TQ-028: Live Tunnel setting changes promote a ready replacement before draining the previous Connector."""

    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.save()
    events = []
    monkeypatch.setattr(remote_access, "status", lambda loaded=None: {"running": True})
    monkeypatch.setattr(remote_access, "_resolve_binary", lambda loaded: "/usr/bin/cloudflared")
    monkeypatch.setattr(
        remote_access,
        "_start_candidate_connector",
        lambda loaded, binary, protocol: events.append(("start", protocol)) or (222, "http://metrics"),
    )
    monkeypatch.setattr(remote_access, "_wait_candidate_ready", lambda pid, metrics: True)

    def promote(pid, **kwargs):
        assert not remote_access.CONFIG_LOCK._is_owned()
        events.append(("promote", kwargs["runtime_signature"]["transport_protocol"]))
        return {"pid": 111}

    monkeypatch.setattr(remote_access, "_promote_candidate_connector", promote)
    monkeypatch.setattr(
        remote_access,
        "_drain_tracked_connector",
        lambda connector: events.append(("drain", connector["pid"])) or True,
    )
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = None

    result = remote_access.apply_settings({"transport_protocol": "http2"})

    assert result["ok"] is True
    assert result["connector_replaced"] is True
    assert V2Config.load().remote_access.vibe_cloud.transport_protocol == "http2"
    assert events == [("start", "http2"), ("promote", "http2"), ("drain", 111)]


@pytest.mark.parametrize(
    "lane_owner",
    ["reserved_recovery", "pending_tail_rollback", "candidate_pid"],
)
def test_ra_tq_028_settings_wait_for_every_candidate_lane_owner(
    monkeypatch,
    tmp_path,
    lane_owner,
) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.save()
    monkeypatch.setattr(remote_access, "status", lambda loaded=None: {"running": True})
    monkeypatch.setattr(remote_access, "_resolve_binary", lambda loaded: "/usr/bin/cloudflared")
    monkeypatch.setattr(
        remote_access,
        "_start_candidate_connector",
        lambda *args, **kwargs: pytest.fail("candidate lane must remain reserved"),
    )

    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = (
            threading.Thread() if lane_owner == "reserved_recovery" else None
        )
    if lane_owner == "pending_tail_rollback":
        runtime.write_json(
            remote_access._state_path(),
            {
                "pending_tail_rollback": {
                    "active_pid": 111,
                    "previous_protocol": "quic",
                }
            },
        )
    elif lane_owner == "candidate_pid":
        remote_access._candidate_pid_path().write_text("222", encoding="utf-8")

    try:
        result = remote_access.apply_settings({"edge_ip_version": "6"})
    finally:
        with remote_access._RECOVERY_LOCK:
            remote_access._RECOVERY_THREAD = None

    assert result["ok"] is False
    assert result["error"] == "remote_access_settings_unavailable"
    assert V2Config.load().remote_access.vibe_cloud.edge_ip_version == "4"


def test_ra_tq_028_settings_do_not_persist_for_unknown_connector_pid(
    monkeypatch,
    tmp_path,
) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.save()
    monkeypatch.setattr(
        remote_access,
        "status",
        lambda loaded=None: {"running": False, "pid": 111, "pid_state": "unknown"},
    )
    monkeypatch.setattr(
        remote_access.api,
        "save_config",
        lambda *args, **kwargs: pytest.fail("unknown connector must keep persisted settings"),
    )

    result = remote_access.apply_settings({"transport_protocol": "http2"})

    assert result["ok"] is False
    assert result["error"] == "remote_access_settings_unavailable"
    assert V2Config.load().remote_access.vibe_cloud.transport_protocol == "auto"


@pytest.mark.parametrize(
    "lane_owner",
    ["reserved_recovery", "pending_tail_rollback", "candidate_pid"],
)
def test_ra_tq_028_settings_do_not_persist_when_connector_lane_is_owned_and_down(
    monkeypatch,
    tmp_path,
    lane_owner,
) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.save()
    monkeypatch.setattr(remote_access, "status", lambda loaded=None: {"running": False})
    monkeypatch.setattr(
        remote_access.api,
        "save_config",
        lambda *args, **kwargs: pytest.fail("owned connector lane must not persist settings"),
    )

    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = (
            threading.Thread() if lane_owner == "reserved_recovery" else None
        )
    if lane_owner == "pending_tail_rollback":
        runtime.write_json(
            remote_access._state_path(),
            {
                "pending_tail_rollback": {
                    "active_pid": 111,
                    "previous_protocol": "quic",
                }
            },
        )
    elif lane_owner == "candidate_pid":
        remote_access._candidate_pid_path().write_text("222", encoding="utf-8")

    try:
        result = remote_access.apply_settings({"edge_ip_version": "6"})
    finally:
        with remote_access._RECOVERY_LOCK:
            remote_access._RECOVERY_THREAD = None

    assert result["ok"] is False
    assert result["error"] == "remote_access_settings_unavailable"
    assert V2Config.load().remote_access.vibe_cloud.edge_ip_version == "4"


def test_ra_tq_028_settings_recheck_stopped_connector_before_persisting(
    monkeypatch,
    tmp_path,
) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.save()
    status_calls = 0
    started = []

    def connector_status(loaded=None):
        nonlocal status_calls
        status_calls += 1
        return {"running": status_calls >= 2, "pid_state": "cloudflared"}

    monkeypatch.setattr(remote_access, "status", connector_status)
    monkeypatch.setattr(remote_access, "_resolve_binary", lambda loaded: "/usr/bin/cloudflared")
    monkeypatch.setattr(
        remote_access,
        "_start_candidate_connector",
        lambda loaded, binary, protocol: started.append(protocol) or (222, "http://metrics"),
    )
    monkeypatch.setattr(remote_access, "_wait_candidate_ready", lambda pid, metrics: False)
    monkeypatch.setattr(remote_access, "_discard_candidate_connector", lambda pid: None)
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = None

    result = remote_access.apply_settings({"transport_protocol": "http2"})

    assert result["ok"] is False
    assert started == ["http2"]
    assert V2Config.load().remote_access.vibe_cloud.transport_protocol == "auto"


def test_ra_tq_028_settings_auto_protocol_falls_back_after_preference_fails(
    monkeypatch,
    tmp_path,
) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.save()
    protocols = []
    discarded = []
    candidate_pids = iter([222, 333])
    monkeypatch.setattr(remote_access, "status", lambda loaded=None: {"running": True})
    monkeypatch.setattr(remote_access, "_resolve_binary", lambda loaded: "/usr/bin/cloudflared")
    monkeypatch.setattr(
        remote_access,
        "_start_candidate_connector",
        lambda loaded, binary, protocol: (
            protocols.append(protocol) or next(candidate_pids),
            f"http://metrics/{protocol}",
        ),
    )
    monkeypatch.setattr(
        remote_access,
        "_wait_candidate_ready",
        lambda pid, metrics: pid == 333,
    )
    monkeypatch.setattr(
        remote_access,
        "_discard_candidate_connector",
        lambda pid: discarded.append(pid),
    )
    monkeypatch.setattr(
        remote_access,
        "_promote_candidate_connector",
        lambda pid, **kwargs: {"pid": 111},
    )
    monkeypatch.setattr(remote_access, "_drain_tracked_connector", lambda connector: True)
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = None
    previous_preference = remote_access._PREFERRED_PROTOCOL
    remote_access._PREFERRED_PROTOCOL = "quic"

    try:
        result = remote_access.apply_settings({"edge_ip_version": "6"})
    finally:
        remote_access._PREFERRED_PROTOCOL = previous_preference

    assert result["ok"] is True
    assert protocols == ["quic", "auto"]
    assert discarded == [222]
    assert V2Config.load().remote_access.vibe_cloud.edge_ip_version == "6"


def test_ra_tq_012_tail_recovery_keeps_better_alternate_protocol(monkeypatch, tmp_path) -> None:
    """RA-TQ-012: Auto mode keeps a materially better alternate protocol."""

    config, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    alternate_pid = 333
    alive.add(alternate_pid)
    previous = {
        **_quality(80),
        "protocol": "quic",
        "request_path": _request_path(202, 900, 1840, slow_rate=0.08),
    }
    candidate_path = _request_path(170, 386, 621)
    spawned_protocols = []
    spawned_pids = iter([candidate_pid, alternate_pid])
    results = []

    def spawn_background(args, pid_path, stdout_name, stderr_name, env=None):
        spawned_protocols.append(env["TUNNEL_TRANSPORT_PROTOCOL"])
        pid = next(spawned_pids)
        pid_path.write_text(str(pid), encoding="utf-8")
        return pid

    def stop_pid(pid, timeout=8):
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "spawn_background", spawn_background)
    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(
        remote_access,
        "_measure_promoted_route",
        lambda cfg, protocol, metrics_url: (
            candidate_path if protocol == "http2" else _request_path(250, 950, 1900, slow_rate=0.08),
            _quality(80),
        ),
    )
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))
    monkeypatch.setattr(remote_access, "_set_preferred_protocol", lambda protocol: None)

    remote_access._run_route_optimization(config, "tail_latency", previous)

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == alternate_pid
    assert state["active"]["requested_protocol"] == "http2"
    assert state["draining"] is None
    assert active_pid not in alive
    assert spawned_protocols == ["quic", "http2"]
    assert results[0]["result"] == "improved"
    assert results[0]["previous_protocol"] == "quic"
    assert results[0]["result_protocol"] == "http2"
    assert results[0]["result_p99"] == 621


def test_ra_tq_013_tail_recovery_rolls_back_protocol(monkeypatch, tmp_path) -> None:
    """RA-TQ-013: Non-improving protocol switch rolls back make-before-break."""

    config, active_pid, first_candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    alternate_pid = 333
    rollback_pid = 444
    alive.add(alternate_pid)
    alive.add(rollback_pid)
    previous_path = _request_path(202, 900, 1840, slow_rate=0.08)
    previous = {**_quality(80), "protocol": "quic", "request_path": previous_path}
    worse_path = _request_path(300, 1200, 2400, slow_rate=0.12)
    spawned_pids = iter([first_candidate_pid, alternate_pid, rollback_pid])
    spawned_protocols = []
    drained = []
    results = []

    def spawn_background(args, pid_path, stdout_name, stderr_name, env=None):
        pid = next(spawned_pids)
        spawned_protocols.append(env["TUNNEL_TRANSPORT_PROTOCOL"])
        pid_path.write_text(str(pid), encoding="utf-8")
        return pid

    def stop_pid(pid, timeout=8):
        drained.append(pid)
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "spawn_background", spawn_background)
    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(
        remote_access,
        "_measure_promoted_route",
        lambda cfg, protocol, metrics_url: (worse_path, _quality(80)),
    )
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))
    monkeypatch.setattr(remote_access, "_set_preferred_protocol", lambda protocol: None)

    remote_access._run_route_optimization(config, "tail_latency", previous)

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == rollback_pid
    assert state["active"]["requested_protocol"] == "quic"
    assert state["candidate"] is None
    assert state["draining"] is None
    assert spawned_protocols == ["quic", "http2", "quic"]
    assert drained == [active_pid, first_candidate_pid, alternate_pid]
    assert results[0]["result"] == "no_improvement"
    assert results[0]["result_protocol"] == "quic"


def test_ra_tq_021_tail_rollback_stays_pending_if_replacement_loses_readiness(
    monkeypatch,
    tmp_path,
) -> None:
    """RA-TQ-021: A rollback replacement that loses readiness remains pending for reconciliation."""

    config, _active_pid, first_candidate_pid, alive = _setup_recovery(
        monkeypatch,
        tmp_path,
        _quality(180),
    )
    alternate_pid = 333
    rollback_pid = 444
    alive.update({alternate_pid, rollback_pid})
    previous_path = _request_path(202, 900, 1840, slow_rate=0.08)
    previous = {**_quality(80), "protocol": "quic", "request_path": previous_path}
    worse_path = _request_path(300, 1200, 2400, slow_rate=0.12)
    spawned_pids = iter([first_candidate_pid, alternate_pid, rollback_pid])
    measurement_count = 0
    rollback_lost_readiness = False
    results = []

    def spawn_background(args, pid_path, stdout_name, stderr_name, env=None):
        pid = next(spawned_pids)
        pid_path.write_text(str(pid), encoding="utf-8")
        return pid

    def measure_promoted_route(cfg, protocol, metrics_url):
        nonlocal measurement_count, rollback_lost_readiness
        measurement_count += 1
        if measurement_count == 3:
            rollback_lost_readiness = True
        return worse_path, _quality(80)

    monkeypatch.setattr(runtime, "spawn_background", spawn_background)
    monkeypatch.setattr(runtime, "stop_pid", lambda pid, timeout=8: alive.discard(pid) is None)
    monkeypatch.setattr(remote_access, "_measure_promoted_route", measure_promoted_route)
    monkeypatch.setattr(
        remote_access,
        "_connector_ready_now",
        lambda pid, url: not (pid == rollback_pid and rollback_lost_readiness),
    )
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    remote_access._run_route_optimization(config, "tail_latency", previous)

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == rollback_pid
    assert state["active"]["requested_protocol"] == "quic"
    assert state["pending_tail_rollback"] == {
        "active_pid": rollback_pid,
        "previous_protocol": "quic",
    }
    assert results[0]["result"] == "failed"


def test_ra_tq_018_tail_candidate_must_pass_connector_error_gates(monkeypatch, tmp_path) -> None:
    """RA-TQ-018: Tail candidates must pass connector availability and error gates."""

    config, active_pid, first_candidate_pid, alive = _setup_recovery(
        monkeypatch,
        tmp_path,
        _quality(180),
    )
    config.remote_access.vibe_cloud.transport_protocol = "quic"
    config.save()
    remote_access._write_state(
        active_pid,
        config,
        "/usr/local/bin/cloudflared",
        "http://127.0.0.1:29001",
        requested_protocol="quic",
    )
    rollback_pid = 333
    alive.add(rollback_pid)
    previous_path = _request_path(202, 900, 1840, slow_rate=0.08)
    previous = {**_quality(80), "protocol": "quic", "request_path": previous_path}
    better_path = _request_path(170, 386, 621)
    bad_connector = {**_quality(80), "request_errors_per_minute": 1}
    measurements = iter([(better_path, bad_connector), (previous_path, _quality(80))])
    spawned_pids = iter([first_candidate_pid, rollback_pid])
    results = []

    def spawn_background(args, pid_path, stdout_name, stderr_name, env=None):
        pid = next(spawned_pids)
        pid_path.write_text(str(pid), encoding="utf-8")
        return pid

    monkeypatch.setattr(runtime, "spawn_background", spawn_background)
    monkeypatch.setattr(runtime, "stop_pid", lambda pid, timeout=8: alive.discard(pid) is None)
    monkeypatch.setattr(
        remote_access,
        "_measure_promoted_route",
        lambda cfg, protocol, metrics_url: next(measurements),
    )
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    remote_access._run_route_optimization(config, "tail_latency", previous)

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == rollback_pid
    assert state["active"]["requested_protocol"] == "quic"
    assert state["pending_tail_rollback"] is None
    assert active_pid not in alive
    assert results[0]["result"] == "no_improvement"


def test_tail_recovery_rechecks_cancellation_before_next_protocol(monkeypatch, tmp_path) -> None:
    config, _active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    previous_path = _request_path(202, 900, 1840, slow_rate=0.08)
    previous = {**_quality(80), "protocol": "quic", "request_path": previous_path}
    spawned_protocols = []
    results = []

    def spawn_background(args, pid_path, stdout_name, stderr_name, env=None):
        spawned_protocols.append(env["TUNNEL_TRANSPORT_PROTOCOL"])
        pid_path.write_text(str(candidate_pid), encoding="utf-8")
        return candidate_pid

    def stop_pid(pid, timeout=8):
        alive.discard(pid)
        return True

    def stop_during_measurement(cfg, protocol, metrics_url):
        assert remote_access.stop(cfg)["ok"] is True
        return _request_path(250, 950, 1900, slow_rate=0.08), _quality(80)

    monkeypatch.setattr(runtime, "spawn_background", spawn_background)
    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(remote_access, "_measure_promoted_route", stop_during_measurement)
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    try:
        remote_access._run_route_optimization(config, "tail_latency", previous)
    finally:
        remote_access._RECOVERY_CANCEL_EVENT.clear()

    assert spawned_protocols == ["quic"]
    assert remote_access._read_pid() is None
    assert results[0]["result"] == "failed"


_UNAVAILABLE_REQUEST_PATH = {
    **_request_path(3500, 3500, 3500, failure_rate=1.0),
    "status": "unavailable",
    "success_count": 0,
}


@pytest.mark.parametrize(
    ("previous", "candidate_protocol", "expected_protocol", "expected_preference"),
    [
        # Partial availability keeps the existing edge-reselection behavior.
        ({"ha_connections": 2, "protocol": "quic"}, "quic", "auto", "quic"),
        # RA-TQ-033: a dead QUIC route is replaced over HTTP/2, whether the
        # connector admits it or still reports a stale ready connection.
        ({"ha_connections": 0, "protocol": "quic"}, "http2", "http2", "http2"),
        (
            {"ha_connections": 1, "protocol": "quic", "request_path": _UNAVAILABLE_REQUEST_PATH},
            "http2",
            "http2",
            "http2",
        ),
        (
            {"ha_connections": 4, "protocol": "quic", "request_path": _UNAVAILABLE_REQUEST_PATH},
            "http2",
            "http2",
            "http2",
        ),
        # Cloudflare auto already starts on QUIC, the alternative to a dead
        # HTTP/2 route; the transport it verified replaces the failed preference.
        ({"ha_connections": 0, "protocol": "http2"}, "quic", "auto", "quic"),
    ],
)
def test_ra_tq_033_availability_recovery_protocol_follows_route_outage(
    monkeypatch,
    tmp_path,
    previous,
    candidate_protocol,
    expected_protocol,
    expected_preference,
) -> None:
    """RA-TQ-033: A route that serves no public requests recovers over the other transport despite cooldown."""

    config, active_pid, candidate_pid, alive = _setup_recovery(
        monkeypatch,
        tmp_path,
        {**_quality(80), "protocol": candidate_protocol},
    )
    remote_access._write_state(
        active_pid,
        config,
        "/usr/local/bin/cloudflared",
        "http://127.0.0.1:29001",
        requested_protocol=previous["protocol"],
    )
    monkeypatch.setattr(remote_access, "_PREFERRED_PROTOCOL", previous["protocol"])
    remote_access._set_preferred_protocol(previous["protocol"])
    spawned_protocols = []
    results = []
    drain_preferences = []

    def spawn_background(args, pid_path, stdout_name, stderr_name, env=None):
        spawned_protocols.append(env["TUNNEL_TRANSPORT_PROTOCOL"])
        pid_path.write_text(str(candidate_pid), encoding="utf-8")
        return candidate_pid

    def stop_pid(pid, timeout=8):
        # The drain can outlive the process, so the preference must already be durable.
        drain_preferences.append(remote_access._stored_preferred_protocol())
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "spawn_background", spawn_background)
    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    remote_access._run_route_optimization(
        config,
        "availability",
        {**_quality(80), **previous},
    )

    assert spawned_protocols == [expected_protocol]
    assert results[0]["result"] == "improved"
    assert drain_preferences == [expected_preference]


@pytest.mark.parametrize(
    ("request_path", "expected_started"),
    [
        # RA-TQ-033: a route serving no public request overrides the cooldown
        # even though cloudflared still reports one ready connection.
        (_UNAVAILABLE_REQUEST_PATH, True),
        (_request_path(900, 1800, 2500, failure_rate=0.5), False),
    ],
)
def test_ra_tq_033_unavailable_route_bypasses_recovery_cooldown(
    monkeypatch,
    tmp_path,
    request_path,
    expected_started,
) -> None:
    started = []

    class FakeThread:
        def __init__(self, **kwargs):
            self.kwargs = kwargs

        def start(self):
            started.append(self)

    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    now = time.time()
    previous = {**_quality(80), "ha_connections": 1, "protocol": "quic", "request_path": request_path}
    monkeypatch.setattr(remote_access, "status", lambda config=None: {"running": True})
    monkeypatch.setattr(remote_access, "_fresh_active_comparison_snapshot", lambda *args, **kwargs: previous)
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)
    monkeypatch.setattr(remote_access.threading, "Thread", FakeThread)
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = None
        remote_access._RECOVERY_ATTEMPTS[:] = [now - 600]
        remote_access._RECOVERY_STATE["next_attempt_at"] = remote_access.tunnel_quality.utc_timestamp(now + 3600)
        remote_access._RECOVERY_EMERGENCY_BYPASS_USED = False

    try:
        result = remote_access.optimize_route(trigger="availability")
    finally:
        with remote_access._RECOVERY_LOCK:
            remote_access._RECOVERY_THREAD = None
            remote_access._RECOVERY_ATTEMPTS.clear()
            remote_access._RECOVERY_STATE.clear()
            remote_access._RECOVERY_STATE.update(remote_access.tunnel_quality.empty_recovery())
            remote_access._RECOVERY_EMERGENCY_BYPASS_USED = False

    assert result["ok"] is expected_started
    assert len(started) == int(expected_started)


def test_optimize_route_reserves_single_candidate_atomically(monkeypatch, tmp_path) -> None:
    started = []
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))

    class FakeThread:
        def __init__(self, **kwargs):
            self.kwargs = kwargs

        def start(self):
            started.append(self)

    monkeypatch.setattr(remote_access, "status", lambda config=None: {"running": True})
    monkeypatch.setattr(remote_access, "_fresh_active_comparison_snapshot", lambda *args, **kwargs: _quality(250))
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)
    monkeypatch.setattr(remote_access.threading, "Thread", FakeThread)
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = None
        remote_access._RECOVERY_STATE.clear()
        remote_access._RECOVERY_STATE.update(remote_access.tunnel_quality.empty_recovery())
        remote_access._RECOVERY_MANUAL_BYPASS_USED = False

    try:
        first = remote_access.optimize_route(trigger="manual")
        second = remote_access.optimize_route(trigger="manual")
    finally:
        with remote_access._RECOVERY_LOCK:
            remote_access._RECOVERY_THREAD = None

    assert first["ok"] is True
    assert second["ok"] is False
    assert second["error"] == "route_optimization_unavailable"
    assert len(started) == 1


def test_zero_connection_recovery_bypasses_attempt_budget_once(monkeypatch) -> None:
    now = time.time()
    first_thread = object()
    second_thread = object()
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = None
        remote_access._RECOVERY_ATTEMPTS[:] = [now - 10, now - 5]
        remote_access._RECOVERY_STATE["next_attempt_at"] = remote_access.tunnel_quality.utc_timestamp(now + 900)
        remote_access._RECOVERY_EMERGENCY_BYPASS_USED = False

    try:
        first = remote_access._reserve_recovery(first_thread, manual=False, emergency=True, now=now)
        with remote_access._RECOVERY_LOCK:
            remote_access._RECOVERY_THREAD = None
        second = remote_access._reserve_recovery(second_thread, manual=False, emergency=True, now=now)
    finally:
        with remote_access._RECOVERY_LOCK:
            remote_access._RECOVERY_THREAD = None
            remote_access._RECOVERY_ATTEMPTS.clear()
            remote_access._RECOVERY_STATE.update(remote_access.tunnel_quality.empty_recovery())
            remote_access._RECOVERY_EMERGENCY_BYPASS_USED = False

    assert first is True
    assert second is False


def test_optimize_route_does_not_start_without_fresh_active_sample(monkeypatch) -> None:
    monkeypatch.setattr(remote_access, "status", lambda config=None: {"running": True})
    monkeypatch.setattr(remote_access, "_fresh_active_comparison_snapshot", lambda *args, **kwargs: None)
    monkeypatch.setattr(
        remote_access.threading,
        "Thread",
        lambda **kwargs: (_ for _ in ()).throw(AssertionError("candidate thread must not start")),
    )

    result = remote_access.optimize_route(trigger="manual")

    assert result["ok"] is False
    assert result["error"] == "route_optimization_unavailable"


@pytest.mark.parametrize(
    ("ready_body", "expected"),
    [
        # cloudflared raises the HA gauge to 4 once the fourth connection starts
        # dialing, while the edge has registered only the first one.
        ({"status": 200, "readyConnections": 1}, False),
        ({"status": 200, "readyConnections": 3}, False),
        ({"status": 200, "readyConnections": 4}, True),
        # A /ready body without the count falls back to the HA gauge.
        ("OK", True),
    ],
)
def test_ra_tq_035_connector_readiness_counts_edge_registered_connections(
    monkeypatch,
    ready_body: dict | str,
    expected: bool,
) -> None:
    """RA-TQ-035: A Connector is ready only after the edge registers all four connections."""

    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

    class CloudflaredMetrics(BaseHTTPRequestHandler):
        def do_GET(self) -> None:  # noqa: N802 - http.server API
            if self.path == "/ready":
                body = json.dumps(ready_body) if isinstance(ready_body, dict) else ready_body
            elif self.path == "/metrics":
                body = "# TYPE cloudflared_tunnel_ha_connections gauge\ncloudflared_tunnel_ha_connections 4\n"
            else:
                self.send_error(404)
                return
            payload = body.encode()
            self.send_response(200)
            self.send_header("Content-Length", str(len(payload)))
            self.end_headers()
            self.wfile.write(payload)

        def log_message(self, *args) -> None:
            pass

    server = ThreadingHTTPServer(("127.0.0.1", 0), CloudflaredMetrics)
    thread = threading.Thread(target=server.serve_forever, daemon=True)
    thread.start()
    monkeypatch.setattr(remote_access, "_is_cloudflared_pid", lambda pid: True)
    try:
        metrics_url = f"http://127.0.0.1:{server.server_address[1]}"
        assert remote_access._connector_ready_now(222, metrics_url) is expected
    finally:
        server.shutdown()
        server.server_close()
        thread.join(timeout=5)


def test_ra_tq_022_public_probe_session_honors_host_network_environment(
    monkeypatch,
    tmp_path,
) -> None:
    """RA-TQ-022: Public probes honor the host proxy and CA environment."""

    ca_bundle = tmp_path / "host-ca.pem"
    ca_bundle.write_text("test CA", encoding="utf-8")
    monkeypatch.setenv("HTTPS_PROXY", "http://proxy.test:8080")
    monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost")
    monkeypatch.setenv("REQUESTS_CA_BUNDLE", str(ca_bundle))
    monkeypatch.delenv("ALL_PROXY", raising=False)
    monkeypatch.delenv("all_proxy", raising=False)

    session = remote_access._public_probe_session()
    try:
        settings = session.merge_environment_settings(
            "https://alex.avibe.bot/health",
            {},
            None,
            None,
            None,
        )
    finally:
        session.close()

    assert session.trust_env is True
    assert settings["proxies"]["https"] == "http://proxy.test:8080"
    assert settings["verify"] == str(ca_bundle)


def test_manual_optimization_uses_active_availability_trigger(monkeypatch) -> None:
    now = time.time()
    quality = {
        **_quality(250),
        "state": "degraded",
        "ha_connections": 3,
        "sampled_at": remote_access.tunnel_quality.utc_timestamp(now),
        "rtt_ms": None,
    }
    started = []

    class FakeThread:
        def __init__(self, **kwargs):
            self.kwargs = kwargs

        def start(self):
            started.append(self)

    monkeypatch.setattr(remote_access, "status", lambda config=None: {"running": True, "tunnel_quality": quality})
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)
    monkeypatch.setattr(remote_access.threading, "Thread", FakeThread)
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = None
        remote_access._RECOVERY_STATE.clear()
        remote_access._RECOVERY_STATE.update(remote_access.tunnel_quality.empty_recovery())

    try:
        result = remote_access.optimize_route(trigger="manual")
    finally:
        with remote_access._RECOVERY_LOCK:
            remote_access._RECOVERY_THREAD = None

    assert result["ok"] is True
    assert len(started) == 1
    assert started[0].kwargs["args"][1] == "availability"


def test_error_recovery_accepts_fresh_http2_snapshot_without_rtt() -> None:
    now = time.time()
    quality = {
        **_quality(250),
        "state": "degraded",
        "sampled_at": remote_access.tunnel_quality.utc_timestamp(now),
        "rtt_ms": None,
        "request_errors_per_minute": 3,
    }

    refreshed = remote_access._fresh_active_comparison_snapshot(
        {"tunnel_quality": quality},
        trigger="errors",
        now=now,
    )

    assert refreshed == quality


def test_stale_active_snapshot_is_refreshed_before_comparison(monkeypatch, tmp_path) -> None:
    _setup_recovery(monkeypatch, tmp_path, _quality(180))
    now = time.time()
    stale = {
        **_quality(250),
        "sampled_at": remote_access.tunnel_quality.utc_timestamp(
            now - remote_access.QUALITY_COMPARISON_MAX_AGE_SECONDS - 1
        ),
    }
    sample = remote_access.tunnel_quality.MetricsSample(
        sampled_at=now,
        ready=True,
        ha_connections=4,
        edge_locations=("sin01", "sin02"),
        smoothed_rtt_ms=(70, 80, 90, 100),
        request_errors_total=0,
        packet_loss_total=0,
        closed_connections_total=0,
    )
    monkeypatch.setattr(remote_access.tunnel_quality, "scrape_metrics", lambda url: sample)

    refreshed = remote_access._fresh_active_comparison_snapshot(
        {"tunnel_quality": stale},
        trigger="manual",
        now=now,
    )

    assert refreshed is not None
    assert refreshed["rtt_ms"]["median"] == 85.0


def test_candidate_exit_before_promotion_keeps_active(monkeypatch, tmp_path) -> None:
    config, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    results = []

    def candidate_exits(_url):
        alive.discard(candidate_pid)
        return _quality(180)

    monkeypatch.setattr(remote_access, "_candidate_average_snapshot", candidate_exits)
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))
    monkeypatch.setattr(runtime, "stop_pid", lambda pid, timeout=8: alive.discard(pid) is None)

    remote_access._run_route_optimization(config, "latency")

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == active_pid
    assert state["active"]["pid"] == active_pid
    assert state["candidate"] is None
    assert results[0]["result"] == "failed"


def test_active_exit_during_evaluation_promotes_stable_candidate(monkeypatch, tmp_path) -> None:
    config, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(220))
    results = []

    def active_exits(_url):
        alive.discard(active_pid)
        return _quality(220)

    monkeypatch.setattr(remote_access, "_candidate_average_snapshot", active_exits)
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    remote_access._run_route_optimization(config, "latency")

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == candidate_pid
    assert state["active"]["pid"] == candidate_pid
    assert state["draining"] is None
    assert results[0]["trigger"] == "availability"
    assert results[0]["result"] == "improved"


def test_failed_drain_remains_tracked_for_reconcile(monkeypatch, tmp_path) -> None:
    config, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    results = []
    monkeypatch.setattr(runtime, "stop_pid", lambda pid, timeout=8: False)
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    remote_access._run_route_optimization(config, "latency")

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert state["active"]["pid"] == candidate_pid
    assert state["draining"]["pid"] == active_pid
    assert results[0]["result"] == "improved"

    def stop_on_reconcile(pid, timeout=8):
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "stop_pid", stop_on_reconcile)
    remote_access._reconcile_draining_connector()
    reconciled = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert reconciled["draining"] is None
    assert active_pid not in alive


def test_ra_tq_016_monitor_lifecycle_retries_failed_drain(monkeypatch, tmp_path) -> None:
    """RA-TQ-016: Failed old-connector drains are retried during monitor lifecycle reconciliation."""

    config, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    state["active"] = remote_access._connector_record(
        candidate_pid,
        "http://127.0.0.1:29002",
        stdout_path=remote_access._candidate_cloudflared_stdout_path(),
        stderr_path=remote_access._candidate_cloudflared_stderr_path(),
    )
    state["draining"] = state["active"] | {
        "pid": active_pid,
        "metrics_url": "http://127.0.0.1:29001",
    }
    state["pid"] = candidate_pid
    runtime.write_json(remote_access._state_path(), state)
    remote_access._pid_path().write_text(str(candidate_pid), encoding="utf-8")
    attempts = iter([False, True])

    def stop_pid(pid, timeout=8):
        stopped = next(attempts)
        if stopped:
            alive.discard(pid)
        return stopped

    monkeypatch.setattr(runtime, "stop_pid", stop_pid)

    remote_access._reconcile_connector_lifecycle()
    assert remote_access._state_connector("draining")["pid"] == active_pid

    remote_access._reconcile_connector_lifecycle()
    assert remote_access._state_connector("draining") is None
    assert active_pid not in alive


def test_monitor_lifecycle_preserves_pending_work_after_transient_error(
    monkeypatch,
    tmp_path,
) -> None:
    _setup_recovery(monkeypatch, tmp_path, _quality(180))
    calls = []
    monkeypatch.setattr(
        remote_access,
        "_reconcile_pending_tail_rollback",
        lambda: (_ for _ in ()).throw(OSError("transient state read")),
    )
    monkeypatch.setattr(remote_access, "_reconcile_draining_connector", lambda: calls.append("drain"))

    remote_access._reconcile_connector_lifecycle()

    assert calls == []


def test_ra_tq_008_restart_removes_orphan_candidate(monkeypatch, tmp_path) -> None:
    """RA-TQ-008: Restart removes an orphan candidate while retaining a healthy active Connector."""

    _, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    state["candidate"] = remote_access._connector_record(
        candidate_pid,
        "http://127.0.0.1:29002",
        stdout_path=remote_access._candidate_cloudflared_stdout_path(),
        stderr_path=remote_access._candidate_cloudflared_stderr_path(),
    )
    runtime.write_json(remote_access._state_path(), state)
    stopped = []

    def stop_pid(pid, timeout=8):
        stopped.append(pid)
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)

    remote_access._reconcile_orphan_candidate()

    reconciled = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == active_pid
    assert reconciled["active"]["pid"] == active_pid
    assert reconciled["candidate"] is None
    assert stopped == [candidate_pid]


def test_ra_tq_017_restart_rolls_back_unverified_tail_candidate(monkeypatch, tmp_path) -> None:
    """RA-TQ-017: Restart rolls back a promoted tail route that was never verified."""

    config, previous_pid, unverified_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    rollback_pid = 333
    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    state["active"] = remote_access._connector_record(
        unverified_pid,
        "http://127.0.0.1:29002",
        stdout_path=remote_access._candidate_cloudflared_stdout_path(),
        stderr_path=remote_access._candidate_cloudflared_stderr_path(),
        requested_protocol="http2",
    )
    state["candidate"] = None
    state["draining"] = None
    state["pending_tail_rollback"] = {
        "active_pid": unverified_pid,
        "previous_protocol": "quic",
    }
    state["pid"] = unverified_pid
    runtime.write_json(remote_access._state_path(), state)
    remote_access._pid_path().write_text(str(unverified_pid), encoding="utf-8")
    alive.discard(previous_pid)
    stopped = []
    results = []

    def spawn_background(args, pid_path, stdout_name, stderr_name, env=None):
        assert env["TUNNEL_TRANSPORT_PROTOCOL"] == "quic"
        alive.add(rollback_pid)
        pid_path.write_text(str(rollback_pid), encoding="utf-8")
        return rollback_pid

    def stop_pid(pid, timeout=8):
        stopped.append(pid)
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "spawn_background", spawn_background)
    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(
        remote_access,
        "_finish_reconciled_recovery",
        lambda result, **kwargs: results.append((result, kwargs)),
    )
    monkeypatch.setattr(remote_access, "_set_preferred_protocol", lambda protocol: None)

    remote_access._reconcile_pending_tail_rollback()

    reconciled = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == rollback_pid
    assert reconciled["active"]["requested_protocol"] == "quic"
    assert reconciled["pending_tail_rollback"] is None
    assert reconciled["draining"] is None
    assert stopped == [unverified_pid]
    assert results == [("failed", {"result_protocol": "quic"})]


def test_ra_tq_019_live_tail_recovery_owns_pending_rollback(monkeypatch, tmp_path) -> None:
    """RA-TQ-019: Live tail recovery retains ownership of its pending rollback marker."""

    _, previous_pid, candidate_pid, _alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    state["active"] = remote_access._connector_record(
        candidate_pid,
        "http://127.0.0.1:29002",
        stdout_path=remote_access._candidate_cloudflared_stdout_path(),
        stderr_path=remote_access._candidate_cloudflared_stderr_path(),
        requested_protocol="http2",
    )
    state["draining"] = None
    state["pending_tail_rollback"] = {
        "active_pid": candidate_pid,
        "previous_protocol": "quic",
    }
    state["pid"] = candidate_pid
    runtime.write_json(remote_access._state_path(), state)
    remote_access._pid_path().write_text(str(candidate_pid), encoding="utf-8")
    calls = []
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_THREAD = object()
    monkeypatch.setattr(
        remote_access,
        "_start_rollback_replacement",
        lambda pending: calls.append(pending) or True,
    )

    try:
        remote_access._reconcile_pending_tail_rollback()
    finally:
        with remote_access._RECOVERY_LOCK:
            remote_access._RECOVERY_THREAD = None

    current = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert current["active"]["pid"] == candidate_pid
    assert current["pending_tail_rollback"]["active_pid"] == candidate_pid
    assert previous_pid != candidate_pid
    assert calls == []


def test_ra_tq_023_status_preserves_rollback_after_promoted_process_exit(
    monkeypatch,
    tmp_path,
) -> None:
    """RA-TQ-023: Status cleanup preserves rollback after a promoted process exits."""

    config, _previous_pid, promoted_pid, alive = _setup_recovery(
        monkeypatch,
        tmp_path,
        _quality(180),
    )
    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    state["active"] = remote_access._connector_record(
        promoted_pid,
        "http://127.0.0.1:29002",
        stdout_path=remote_access._candidate_cloudflared_stdout_path(),
        stderr_path=remote_access._candidate_cloudflared_stderr_path(),
        requested_protocol="http2",
    )
    state["candidate"] = None
    state["draining"] = None
    state["pending_tail_rollback"] = {
        "active_pid": promoted_pid,
        "previous_protocol": "quic",
    }
    state["pid"] = promoted_pid
    runtime.write_json(remote_access._state_path(), state)
    remote_access._pid_path().write_text(str(promoted_pid), encoding="utf-8")
    alive.discard(promoted_pid)
    rollback_attempts = []
    monkeypatch.setattr(
        remote_access,
        "_start_rollback_replacement",
        lambda pending: rollback_attempts.append(pending) or False,
    )

    current = remote_access.status(config)
    remote_access._reconcile_pending_tail_rollback()

    preserved = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert current["running"] is False
    assert current["pid"] is None
    assert preserved["active"]["pid"] == promoted_pid
    assert preserved["pending_tail_rollback"] == {
        "active_pid": promoted_pid,
        "previous_protocol": "quic",
    }
    assert rollback_attempts == [preserved["pending_tail_rollback"]]


def test_ra_tq_024_pinned_protocol_rejects_stale_tail_evidence(
    monkeypatch,
    tmp_path,
) -> None:
    """RA-TQ-024: Pinned transport rejects stale comparison evidence from another protocol."""

    config, active_pid, _candidate_pid, _alive = _setup_recovery(
        monkeypatch,
        tmp_path,
        _quality(180),
    )
    config.remote_access.vibe_cloud.transport_protocol = "http2"
    config.save()
    remote_access._write_state(
        active_pid,
        config,
        "/usr/local/bin/cloudflared",
        "http://127.0.0.1:29001",
        requested_protocol="http2",
    )
    now = time.time()
    stale = {
        **_quality(80),
        "protocol": "quic",
        "sampled_at": remote_access.tunnel_quality.utc_timestamp(now),
        "request_path": _request_path(202, 900, 1840, slow_rate=0.08),
    }
    current = {
        "transport_protocol": "http2",
        "tunnel_quality": stale,
    }

    assert (
        remote_access._fresh_active_comparison_snapshot(
            current,
            trigger="tail_latency",
            now=now,
        )
        is None
    )
    with pytest.raises(RuntimeError, match="request_path_comparison_unavailable"):
        remote_access._run_tail_protocol_recovery(
            config,
            "/usr/local/bin/cloudflared",
            stale,
        )


def test_restart_removes_candidate_recorded_only_in_pid_file(monkeypatch, tmp_path) -> None:
    _, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    remote_access._candidate_pid_path().write_text(str(candidate_pid), encoding="utf-8")
    stopped = []

    def stop_pid(pid, timeout=8):
        stopped.append(pid)
        alive.discard(pid)
        return True

    monkeypatch.setattr(runtime, "stop_pid", stop_pid)
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)

    remote_access._reconcile_orphan_candidate()

    reconciled = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == active_pid
    assert reconciled["active"]["pid"] == active_pid
    assert reconciled["candidate"] is None
    assert not remote_access._candidate_pid_path().exists()
    assert stopped == [candidate_pid]


def test_restart_clears_candidate_pid_when_it_is_already_active(monkeypatch, tmp_path) -> None:
    _, active_pid, _, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    remote_access._pid_path().write_text("999", encoding="utf-8")
    remote_access._candidate_pid_path().write_text(str(active_pid), encoding="utf-8")
    stopped = []
    recovery_results = []
    monkeypatch.setattr(runtime, "stop_pid", lambda pid, timeout=8: stopped.append(pid) or True)
    monkeypatch.setattr(remote_access, "_finish_reconciled_recovery", lambda result: recovery_results.append(result))

    remote_access._reconcile_orphan_candidate()

    assert active_pid in alive
    assert stopped == []
    assert remote_access._read_pid() == active_pid
    assert not remote_access._candidate_pid_path().exists()
    assert recovery_results == ["improved"]


def test_failed_candidate_with_unknown_identity_remains_tracked(monkeypatch, tmp_path) -> None:
    config, active_pid, candidate_pid, _ = _setup_recovery(monkeypatch, tmp_path, _quality(220))
    results = []
    monkeypatch.setattr(
        runtime,
        "get_process_command",
        lambda pid: "/usr/local/bin/cloudflared tunnel run" if pid == active_pid else None,
    )
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: results.append(result))

    remote_access._run_route_optimization(config, "latency")

    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert state["candidate"]["pid"] == candidate_pid
    assert remote_access._candidate_pid_path().read_text(encoding="utf-8") == str(candidate_pid)
    assert results[0]["result"] == "no_improvement"
    assert remote_access._reserve_recovery(object(), manual=False, emergency=False, now=time.time()) is False


def test_startup_resets_inflight_recovery_without_connector(monkeypatch, tmp_path) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_STATE.clear()
        remote_access._RECOVERY_STATE.update(
            {
                **remote_access.tunnel_quality.empty_recovery(),
                "state": "evaluating",
                "last_trigger": "errors",
            }
        )
    monkeypatch.setattr(remote_access, "_report_runtime_status_async", lambda *args, **kwargs: None)

    remote_access._normalize_orphaned_recovery_state()

    recovery = remote_access._recovery_payload()
    assert recovery["state"] == "cooldown"
    assert recovery["last_result"] == "failed"
    assert recovery["last_trigger"] == "errors"
    assert recovery["next_attempt_at"] is not None


def test_recovery_completion_restarts_healthy_reset_window(monkeypatch) -> None:
    evaluator = remote_access.tunnel_quality.QualityEvaluator()
    for index in range(120):
        evaluator.update(
            remote_access.tunnel_quality.MetricsSample(
                sampled_at=100 + index * 15,
                ready=True,
                ha_connections=4,
                edge_locations=("sin01", "sin02"),
                smoothed_rtt_ms=(70, 75, 80, 85),
                request_errors_total=0,
                packet_loss_total=0,
                closed_connections_total=0,
            )
        )
    monkeypatch.setattr(remote_access, "_QUALITY_EVALUATOR", evaluator)
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)
    monkeypatch.setattr(remote_access, "_report_runtime_status_async", lambda *args, **kwargs: None)

    remote_access._finish_recovery(
        trigger="manual",
        result="no_improvement",
        previous_median=77.5,
        result_median=77.5,
    )

    assert evaluator.healthy_samples == 0


def test_reconciled_recovery_uses_only_contract_trigger_values(monkeypatch) -> None:
    captured = []
    with remote_access._RECOVERY_LOCK:
        remote_access._RECOVERY_STATE.clear()
        remote_access._RECOVERY_STATE.update(remote_access.tunnel_quality.empty_recovery())
    monkeypatch.setattr(remote_access, "_finish_recovery", lambda **result: captured.append(result))

    remote_access._finish_reconciled_recovery("failed")

    assert captured[0]["trigger"] is None


def test_ra_tq_009_restart_promotes_ready_candidate(monkeypatch, tmp_path) -> None:
    """RA-TQ-009: Restart promotes a ready candidate when the previous active Connector is gone."""

    _, active_pid, candidate_pid, alive = _setup_recovery(monkeypatch, tmp_path, _quality(180))
    state = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    state["candidate"] = remote_access._connector_record(
        candidate_pid,
        "http://127.0.0.1:29002",
        stdout_path=remote_access._candidate_cloudflared_stdout_path(),
        stderr_path=remote_access._candidate_cloudflared_stderr_path(),
    )
    runtime.write_json(remote_access._state_path(), state)
    alive.discard(active_pid)

    class ReadyResponse:
        ok = True

    monkeypatch.setattr(remote_access.requests, "get", lambda *args, **kwargs: ReadyResponse())
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)

    remote_access._reconcile_orphan_candidate()

    reconciled = json.loads(remote_access._state_path().read_text(encoding="utf-8"))
    assert remote_access._read_pid() == candidate_pid
    assert reconciled["active"]["pid"] == candidate_pid
    assert reconciled["candidate"] is None


def _setup_exiting_connector(monkeypatch, tmp_path) -> tuple[V2Config, Path]:
    """A real connector process that exits about half a second after start while the network is down."""

    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    network_up = tmp_path / "network-up"
    binary = tmp_path / "bin" / "cloudflared"
    binary.parent.mkdir()
    binary.write_text(
        "#!/bin/sh\n"
        f'if [ -e "{network_up}" ]; then while :; do sleep 0.2; done; fi\n'
        'echo "failed to dial to edge: lookup region1.v2.argotunnel.com: no such host" >&2\n'
        "sleep 0.5\n"
        "exit 1\n",
        encoding="utf-8",
    )
    binary.chmod(0o755)
    config = _config()
    config.remote_access.vibe_cloud.tunnel_token = "tunnel-token"
    config.remote_access.vibe_cloud.cloudflared_path = str(binary)
    config.save()
    monkeypatch.setattr(remote_access, "_PREFERRED_PROTOCOL", None)
    monkeypatch.setattr(remote_access, "_version", lambda path: "cloudflared version test")
    monkeypatch.setattr(remote_access, "_report_runtime_status_async", lambda *args, **kwargs: None)

    def process_command(pid: int) -> str | None:
        # The macOS fallback runs setuid ``ps``, which a sandboxed test cannot exec.
        try:
            return " ".join(psutil.Process(pid).cmdline()) or None
        except psutil.Error:
            return None

    monkeypatch.setattr(runtime, "get_process_command", process_command)
    return config, network_up


def _wait_connector_exit(timeout: float = 5.0) -> None:
    deadline = time.monotonic() + timeout
    while remote_access._cloudflared_pid_state(remote_access._read_pid()) not in {"dead", "other"}:
        assert time.monotonic() < deadline, "connector did not exit"
        time.sleep(0.05)


def test_ra_tq_034_connector_exited_before_network_is_restarted_with_backoff(monkeypatch, tmp_path) -> None:
    """RA-TQ-034: A requested Connector that exits on its own restarts with backoff; a stopped or disabled one stays down."""

    config, network_up = _setup_exiting_connector(monkeypatch, tmp_path)
    schedule = remote_access._ConnectorRestartSchedule()
    try:
        # UI start after a reboot: the network is down, so the connector exits within a second.
        remote_access.start()
        _wait_connector_exit()

        remote_access._restart_exited_connector(config, schedule, 1000.0)
        assert not remote_access._is_cloudflared_pid(remote_access._read_pid())
        remote_access._restart_exited_connector(config, schedule, 1000.0 + remote_access.QUALITY_SAMPLE_SECONDS)
        assert schedule.delay_seconds == 2 * remote_access.QUALITY_SAMPLE_SECONDS
        _wait_connector_exit()

        network_up.touch()
        now = 1100.0
        remote_access._restart_exited_connector(config, schedule, now)
        remote_access._restart_exited_connector(config, schedule, now + schedule.delay_seconds - 1)
        assert not remote_access._is_cloudflared_pid(remote_access._read_pid())
        remote_access._restart_exited_connector(config, schedule, now + schedule.delay_seconds)

        pid = remote_access._read_pid()
        time.sleep(1.0)  # longer than the offline connector lives
        assert remote_access._is_cloudflared_pid(pid)
        remote_access._restart_exited_connector(config, schedule, now + 1000)
        assert remote_access._read_pid() == pid
        assert schedule.due_at is None
    finally:
        remote_access.stop()


@pytest.mark.parametrize("withdrawal", ["user_stop", "remote_access_disabled"])
def test_ra_tq_034_withdrawn_connector_request_is_not_restarted(monkeypatch, tmp_path, withdrawal) -> None:
    config, network_up = _setup_exiting_connector(monkeypatch, tmp_path)
    schedule = remote_access._ConnectorRestartSchedule()
    real_start = remote_access._start
    started = []
    try:
        remote_access.start()
        _wait_connector_exit()
        network_up.touch()
        if withdrawal == "user_stop":
            remote_access.stop()
        else:
            config.remote_access.vibe_cloud.enabled = False
            config.save()

        def record_start(loaded, *, record_request, install_failure=None):
            started.append(record_request)
            return real_start(loaded, record_request=record_request, install_failure=install_failure)

        monkeypatch.setattr(remote_access, "_start", record_start)
        for now in (1000.0, 2000.0, 3000.0):
            remote_access._restart_exited_connector(config, schedule, now)

        assert started == []
        assert not remote_access._is_cloudflared_pid(remote_access._read_pid())
    finally:
        remote_access.stop()


def test_ra_tq_034_restart_takes_recovery_lock_before_connector_lock(monkeypatch, tmp_path) -> None:
    config, network_up = _setup_exiting_connector(monkeypatch, tmp_path)
    schedule = remote_access._ConnectorRestartSchedule()
    recovery_holds_its_lock = threading.Event()
    connector_lock_acquired = []

    def reserve_recovery() -> None:
        # The order route recovery and settings use: _RECOVERY_LOCK, then _CONNECTOR_LOCK.
        with remote_access._RECOVERY_LOCK:
            recovery_holds_its_lock.set()
            acquired = remote_access._CONNECTOR_LOCK.acquire(timeout=5)
            connector_lock_acquired.append(acquired)
            if acquired:
                remote_access._CONNECTOR_LOCK.release()

    recovery = threading.Thread(target=reserve_recovery)
    real_read_pid = remote_access._read_pid

    def read_pid_while_recovery_reserves():
        # Recovery reserves the lane right after the restart takes the connector lock.
        if remote_access._CONNECTOR_LOCK._is_owned() and not recovery_holds_its_lock.is_set():
            recovery.start()
            recovery_holds_its_lock.wait(5)
        return real_read_pid()

    try:
        remote_access.start()
        _wait_connector_exit()
        network_up.touch()
        remote_access._restart_exited_connector(config, schedule, 1000.0)
        monkeypatch.setattr(remote_access, "_read_pid", read_pid_while_recovery_reserves)

        remote_access._restart_exited_connector(config, schedule, 1000.0 + schedule.delay_seconds)
        recovery.join(10)

        assert connector_lock_acquired == [True]
    finally:
        monkeypatch.setattr(remote_access, "_read_pid", real_read_pid)
        remote_access.stop()


class _StopMonitor(Exception):
    pass


def test_ra_tq_034_restart_tick_does_not_evaluate_the_exited_connector(monkeypatch, tmp_path) -> None:
    _, network_up = _setup_exiting_connector(monkeypatch, tmp_path)
    interval = remote_access.QUALITY_SAMPLE_SECONDS
    clock = {"now": 1000.0}

    def sleep(seconds: float) -> None:
        clock["now"] += seconds
        if clock["now"] >= 1000.0 + 2 * interval:
            raise _StopMonitor  # end of the second tick

    monkeypatch.setattr(
        remote_access,
        "time",
        type("Clock", (), {"monotonic": staticmethod(lambda: clock["now"]), "sleep": staticmethod(sleep), "time": staticmethod(time.time)}),
    )
    monkeypatch.setattr(remote_access, "_public_health_url", lambda loaded: None)
    # Outage evidence left by the exited connector would fire availability recovery.
    monkeypatch.setattr(remote_access._QUALITY_EVALUATOR, "recovery_trigger", lambda snapshot, profile: "availability")
    optimized_while_running = []
    monkeypatch.setattr(
        remote_access,
        "optimize_route",
        lambda **kwargs: optimized_while_running.append(remote_access._is_cloudflared_pid(remote_access._read_pid())),
    )
    try:
        remote_access.start()
        _wait_connector_exit()
        network_up.touch()
        remote_access._quality_state_path().parent.mkdir(parents=True, exist_ok=True)

        # Tick 1 sees the exit and schedules the restart; tick 2 restarts.
        with pytest.raises(_StopMonitor):
            remote_access._quality_monitor_loop(interval, remote_access._quality_state_path())

        assert remote_access._is_cloudflared_pid(remote_access._read_pid())
        assert optimized_while_running == [False]
    finally:
        remote_access.stop()


def test_ra_tq_034_failed_config_read_during_restart_keeps_the_request(monkeypatch, tmp_path) -> None:
    config, network_up = _setup_exiting_connector(monkeypatch, tmp_path)
    schedule = remote_access._ConnectorRestartSchedule()
    real_load = remote_access.V2Config.load
    reads_under_lock = []

    def load_failing_once_under_connector_lock(*args, **kwargs):
        if remote_access._CONNECTOR_LOCK._is_owned():
            reads_under_lock.append(True)
            # The restart's _start() reads the config twice under the lock;
            # the second read, after its fallback read, fails once.
            if len(reads_under_lock) == 2:
                raise OSError("transient config read failure")
        return real_load(*args, **kwargs)

    try:
        remote_access.start()
        _wait_connector_exit()
        network_up.touch()
        monkeypatch.setattr(remote_access.V2Config, "load", load_failing_once_under_connector_lock)
        remote_access._restart_exited_connector(config, schedule, 1000.0)
        remote_access._restart_exited_connector(config, schedule, 1000.0 + schedule.delay_seconds)
        assert len(reads_under_lock) == 2
        assert remote_access._connector_requested_path().exists()

        now = 2000.0
        remote_access._restart_exited_connector(config, schedule, now)
        remote_access._restart_exited_connector(config, schedule, now + schedule.delay_seconds)
        assert remote_access._is_cloudflared_pid(remote_access._read_pid())
    finally:
        monkeypatch.setattr(remote_access.V2Config, "load", real_load)
        remote_access.stop()


def test_ra_tq_034_stop_still_stops_when_the_request_cannot_be_withdrawn(monkeypatch, tmp_path) -> None:
    config, network_up = _setup_exiting_connector(monkeypatch, tmp_path)
    network_up.touch()
    request = remote_access._connector_requested_path()
    try:
        remote_access.start()
        pid = remote_access._read_pid()
        assert remote_access._is_cloudflared_pid(pid)
        request.unlink()
        request.mkdir()  # a corrupted request that unlink() cannot remove

        result = remote_access.stop()

        assert result["ok"] is False
        assert result["error"] == "remote_access_request_withdraw_failed"
        assert not remote_access._is_cloudflared_pid(pid)
        # What stop could not remove is no request, so the monitor keeps it down.
        schedule = remote_access._ConnectorRestartSchedule()
        for now in (1000.0, 2000.0, 3000.0):
            assert remote_access._restart_exited_connector(config, schedule, now) is False
    finally:
        if request.is_dir():
            request.rmdir()
        remote_access.stop()


_LIFECYCLE_CHILD = """
import json, sys
from pathlib import Path

from tests import process_reaper

process_reaper.record_spawns(Path(sys.argv[3]))

import psutil

from vibe import remote_access, runtime

runtime.get_process_command = lambda pid: (" ".join(psutil.Process(pid).cmdline()) or None) if psutil.pid_exists(pid) else None
remote_access._version = lambda path: "cloudflared version test"
remote_access._report_runtime_status_async = lambda *args, **kwargs: None

action, report = sys.argv[1], Path(sys.argv[2])
if action == "hold":
    with remote_access._connector_lifecycle():
        report.write_text("held", encoding="utf-8")
        sys.stdin.read()  # held until the test kills this process
    sys.exit(0)
try:
    with remote_access._connector_lifecycle(timeout=0):
        probe = "free"
except remote_access._ConnectorLifecycleBusy:
    probe = "busy"
report.with_suffix(".probe").write_text(probe, encoding="utf-8")
result = remote_access.stop() if action == "stop" else remote_access.start()
report.write_text(json.dumps({"ok": result.get("ok"), "error": result.get("error")}), encoding="utf-8")
"""


def _lifecycle_child(tmp_path: Path, action: str, name: str) -> tuple[subprocess.Popen, Path]:
    report = tmp_path / f"{name}.json"
    process = subprocess.Popen(
        [sys.executable, "-c", _LIFECYCLE_CHILD, action, str(report), str(process_reaper.helper_spawn_record(tmp_path))],
        cwd=str(Path(remote_access.__file__).resolve().parents[1]),
        env={**os.environ, "AVIBE_HOME": str(tmp_path)},
        stdin=subprocess.PIPE,
    )
    return process, report


def _wait_for_file(path: Path, timeout: float = 20.0) -> str:
    deadline = time.monotonic() + timeout
    while not path.exists() or not path.read_text(encoding="utf-8"):
        assert time.monotonic() < deadline, f"{path.name} never appeared"
        time.sleep(0.05)
    return path.read_text(encoding="utf-8")


def _live_connectors(binary: Path) -> list[int]:
    live = []
    for process in psutil.process_iter(["pid", "cmdline", "status"]):
        cmdline = process.info.get("cmdline") or []
        if str(binary) in cmdline and process.info.get("status") != psutil.STATUS_ZOMBIE:
            live.append(process.info["pid"])
    return live


@pytest.mark.parametrize("children", [("stop",), ("stop", "start")])
def test_ra_tq_034_cross_process_stop_and_start_serialize_with_the_restart(monkeypatch, tmp_path, children) -> None:
    config, network_up = _setup_exiting_connector(monkeypatch, tmp_path)
    binary = Path(config.remote_access.vibe_cloud.cloudflared_path)
    schedule = remote_access._ConnectorRestartSchedule()
    real_start = remote_access._start
    launched = []

    def start_while_other_processes_wait(loaded, *, record_request, install_failure=None):
        # The restart holds the lifecycle lock here, after its request check and
        # before its spawn: the round-8 ordering would have the stop read the
        # exited pid now. Each child finds the lock busy and must wait for it.
        for name in children:
            launched.append(_lifecycle_child(tmp_path, name, name))
        for _process, report in launched:
            assert _wait_for_file(report.with_suffix(".probe")) == "busy"
        return real_start(loaded, record_request=record_request, install_failure=install_failure)

    try:
        remote_access.start()
        _wait_connector_exit()
        network_up.touch()
        monkeypatch.setattr(remote_access, "_start", start_while_other_processes_wait)
        remote_access._restart_exited_connector(config, schedule, 1000.0)
        remote_access._restart_exited_connector(config, schedule, 1000.0 + schedule.delay_seconds)
        monkeypatch.setattr(remote_access, "_start", real_start)
        for process, report in launched:
            assert process.wait(timeout=60) == 0
            assert json.loads(_wait_for_file(report))["ok"] is True

        live = _live_connectors(binary)
        if remote_access._connector_requested_path().exists():
            # A start won the last word: exactly the recorded connector serves.
            assert live == [remote_access._read_pid()]
        else:
            # The stop won the last word: nothing serves, and the monitor keeps it down.
            assert live == []
            remote_access._restart_exited_connector(config, schedule, 5000.0)
            remote_access._restart_exited_connector(config, schedule, 9000.0)
            assert _live_connectors(binary) == []
    finally:
        for process, _report in launched:
            if process.poll() is None:
                process.kill()
                process.wait()
        monkeypatch.setattr(remote_access, "_start", real_start)
        remote_access.stop()


def test_ra_tq_034_lifecycle_lock_frees_when_its_holder_dies_and_status_never_waits(monkeypatch, tmp_path) -> None:
    _setup_exiting_connector(monkeypatch, tmp_path)
    holder, report = _lifecycle_child(tmp_path, "hold", "holder")
    dead = subprocess.Popen([sys.executable, "-c", "pass"])
    dead.wait()
    try:
        assert _wait_for_file(report) == "held"
        remote_access._pid_path().parent.mkdir(parents=True, exist_ok=True)
        remote_access._pid_path().write_text(str(dead.pid), encoding="utf-8")
        with pytest.raises(remote_access._ConnectorLifecycleBusy):
            with remote_access._connector_lifecycle(timeout=0):
                pass

        started_at = time.monotonic()
        result = remote_access.status()
        assert time.monotonic() - started_at < remote_access.CONNECTOR_LIFECYCLE_WAIT_SECONDS / 3
        assert result["running"] is False
        # The holder owns the records, so status() left them for it.
        assert remote_access._read_pid() == dead.pid

        holder.kill()
        holder.wait()
        with remote_access._connector_lifecycle(timeout=0):
            pass
        remote_access.status()
        assert remote_access._read_pid() is None
    finally:
        if holder.poll() is None:
            holder.kill()
            holder.wait()


def test_ra_tq_034_missing_cloudflared_downloads_outside_the_lifecycle_lock(monkeypatch, tmp_path) -> None:
    config, _ = _setup_exiting_connector(monkeypatch, tmp_path)
    config.remote_access.vibe_cloud.cloudflared_path = ""
    config.save()
    monkeypatch.setattr(remote_access, "_cloudflared_pin_needed", lambda loaded=None: True)
    monkeypatch.setattr(remote_access, "_resolve_binary", lambda loaded=None: None)
    downloads = []

    def install_cloudflared():
        # Another thread can take the lock only if this download does not hold it.
        free = []

        def probe():
            try:
                with remote_access._connector_lifecycle(timeout=0):
                    free.append(True)
            except remote_access._ConnectorLifecycleBusy:
                free.append(False)

        prober = threading.Thread(target=probe)
        prober.start()
        prober.join()
        downloads.append(free[0])
        return {"ok": False, "error": "cloudflared_install_failed", "detail": "offline"}

    monkeypatch.setattr(remote_access, "install_cloudflared", install_cloudflared)

    result = remote_access.start()
    remote_access._restart_exited_connector(config, remote_access._ConnectorRestartSchedule(due_at=0.0), 1.0)

    assert result["ok"] is False and result["detail"] == "offline"
    # One download per start or restart, each outside the lock; none under it.
    assert downloads == [True, True]
    assert remote_access._connector_requested_path().is_file()


def test_ra_tq_034_orphan_candidate_reconcile_waits_for_the_lock_and_for_recovery(monkeypatch, tmp_path) -> None:
    _setup_exiting_connector(monkeypatch, tmp_path)
    dead = subprocess.Popen([sys.executable, "-c", "pass"])
    dead.wait()
    remote_access._state_path().parent.mkdir(parents=True, exist_ok=True)
    runtime.write_json(remote_access._state_path(), {"schema_version": 2, "candidate": {"pid": dead.pid}})
    monkeypatch.setattr(remote_access, "_set_recovery_state", lambda **changes: changes)
    holder, report = _lifecycle_child(tmp_path, "hold", "holder")
    try:
        assert _wait_for_file(report) == "held"
        assert remote_access._reconcile_orphan_candidate_when_free() is False
        holder.kill()
        holder.wait()

        monkeypatch.setattr(remote_access, "_RECOVERY_THREAD", threading.current_thread())
        assert remote_access._reconcile_orphan_candidate_when_free() is False
        assert remote_access._state_connector("candidate") == {"pid": dead.pid}

        monkeypatch.setattr(remote_access, "_RECOVERY_THREAD", None)
        assert remote_access._reconcile_orphan_candidate_when_free() is True
        assert remote_access._state_connector("candidate") is None
    finally:
        if holder.poll() is None:
            holder.kill()
            holder.wait()


def test_ra_tq_034_candidate_cleanup_waits_out_lifecycle_contention(monkeypatch, tmp_path) -> None:
    _setup_exiting_connector(monkeypatch, tmp_path)
    # A short user bound shows the cleanup does not give up when a user step would.
    monkeypatch.setattr(remote_access, "CONNECTOR_LIFECYCLE_WAIT_SECONDS", 0.2)
    dead = subprocess.Popen([sys.executable, "-c", "pass"])
    dead.wait()
    remote_access._state_path().parent.mkdir(parents=True, exist_ok=True)
    runtime.write_json(remote_access._state_path(), {"schema_version": 2, "candidate": {"pid": dead.pid}})
    holder, report = _lifecycle_child(tmp_path, "hold", "holder")
    errors = []

    def discard():
        try:
            remote_access._discard_candidate_connector(dead.pid)
        except BaseException as exc:  # noqa: BLE001
            errors.append(exc)

    cleaner = threading.Thread(target=discard)
    try:
        assert _wait_for_file(report) == "held"
        cleaner.start()
        cleaner.join(timeout=1.0)
        assert cleaner.is_alive() and errors == []  # still waiting, not failed
        holder.kill()
        holder.wait()
        cleaner.join(timeout=10)
        assert not cleaner.is_alive() and errors == []
        assert remote_access._state_connector("candidate") is None
    finally:
        if holder.poll() is None:
            holder.kill()
            holder.wait()
        cleaner.join(timeout=10)


def test_ra_tq_034_start_survives_an_unwritable_request(monkeypatch, tmp_path) -> None:
    _, network_up = _setup_exiting_connector(monkeypatch, tmp_path)
    network_up.touch()
    blocker = tmp_path / "not-a-directory"
    blocker.write_text("", encoding="utf-8")
    # The request's parent is a file, so recording the request raises OSError.
    monkeypatch.setattr(remote_access, "_connector_requested_path", lambda: blocker / "request")
    try:
        result = remote_access.start()

        assert result["ok"] is True and result["started"] is True
        assert remote_access._is_cloudflared_pid(result["pid"])
    finally:
        remote_access.stop()


def test_ra_tq_034_concurrent_starts_share_one_cloudflared_download(monkeypatch, tmp_path) -> None:
    config, _ = _setup_exiting_connector(monkeypatch, tmp_path)
    installed = threading.Event()
    both_checked = threading.Event()
    checks = []
    downloads = []

    def pin_needed(loaded=None):
        checks.append(threading.current_thread().name)
        if len(checks) >= 2:
            both_checked.set()
        return not installed.is_set()

    def install_cloudflared():
        downloads.append(threading.current_thread().name)
        both_checked.wait(5)  # the other start has found cloudflared missing too
        installed.set()
        return {"ok": True, "path": "/fake/cloudflared"}

    monkeypatch.setattr(remote_access, "_cloudflared_pin_needed", pin_needed)
    monkeypatch.setattr(remote_access, "install_cloudflared", install_cloudflared)

    def download():
        remote_access._ensure_cloudflared_pin(config)

    threads = [threading.Thread(target=download, name=f"start-{i}") for i in range(2)]
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join(timeout=10)

    assert len(downloads) == 1


def test_ra_tq_034_candidate_cleanup_gives_up_only_after_its_long_bound(monkeypatch, tmp_path, caplog) -> None:
    _setup_exiting_connector(monkeypatch, tmp_path)
    monkeypatch.setattr(remote_access, "CLEANUP_LOCK_WAIT_SECONDS", 0.3)
    dead = subprocess.Popen([sys.executable, "-c", "pass"])
    dead.wait()
    remote_access._state_path().parent.mkdir(parents=True, exist_ok=True)
    runtime.write_json(remote_access._state_path(), {"schema_version": 2, "candidate": {"pid": dead.pid}})
    holder, report = _lifecycle_child(tmp_path, "hold", "holder")
    try:
        assert _wait_for_file(report) == "held"

        remote_access._discard_candidate_connector(dead.pid)  # returns instead of hanging or raising

        assert remote_access._state_connector("candidate") == {"pid": dead.pid}
        assert "records stay after the lifecycle lock stayed busy" in caplog.text
    finally:
        holder.kill()
        holder.wait()