Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9971: capture delayed download 5xx responses (#8523)
LawyZhengd1b3c3ec3af3
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 571b8c3ae..a17f90583 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -4012,6 +4012,14 @@ class AutoCompletionResult(BaseModel):# Upstream HTTP statuses that count as download-failure evidence; adjacent 5xx (501/505/507/...) are excluded._OBSERVED_DOWNLOAD_FAILURE_STATUSES = frozenset({500, 502, 503, 504})+# Hard cap on the per-action set of xhr/fetch requests eligible for passive status observation. Bounds+# retained request-object references under a request storm; at the cap we stop tracking new requests+# (logging once) instead of growing without limit.+_MAX_STATUS_OBSERVATION_REQUESTS = 64+# Grace after seal_in_flight_requests() during which a newly initiated xhr/fetch can still enter status+# observation. Bounds the observation window to the action plus a JS-initiated second hop, so an unrelated+# heartbeat/poll fired minutes into the long download-wait loop can never stamp download_failure_status.+_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS = 2.0class ScopedXhrDownloadCapture:@@ -4030,24 +4038,44 @@ class ScopedXhrDownloadCapture:page: Page,download_dir: Path,timeout_seconds: float = BROWSER_DOWNLOAD_TIMEOUT,+ monotonic: Callable[[], float] = time.monotonic,) -> None:self._page = pageself._download_dir = download_dirself._timeout_seconds = timeout_seconds+ self._monotonic = monotonicself._saved: set[str] = set()self._extra_pages: list[Page] = []self._capture_responses = Falseself._active = Falseself._accept_new_requests = False# Passive, site-agnostic status observation: records only a server 5xx integer status for any- # admitted xhr/fetch request during this download action. No URL, host, header, body, or other- # request detail is ever retained.+ # xhr/fetch request initiated during this download action's window. No URL, host, header, body,+ # or other request detail is ever retained.self._observed_download_failure_status: int | None = None# Response-body drain count is separate from request-lifecycle tracking for the bounded wait extension.self._in_flight = 0self._response_tasks: set[asyncio.Task[None]] = set()self._in_flight_requests: set[Request] = set()self._admitted_requests: set[Request] = set()+ # Requests initiated during the active window (seen by _on_request after enable), used ONLY to+ # scope passive status observation. Deliberately independent of _admitted_requests/seal so a+ # JS-initiated second hop that starts after seal is still observed, while a request already in+ # flight before this action (never seen by _on_request) is excluded. Bounded by+ # _MAX_STATUS_OBSERVATION_REQUESTS; never gates wait extension.+ self._status_observation_requests: set[Request] = set()+ # Child pages whose bootstrap provenance was established while inside the observation window.+ # A bootstrap hop on such a page stays observation-eligible after the deadline (proven+ # action-owned); a child page first seen at/after the deadline is admitted for wait extension+ # but never grants observation provenance, so its 5xx cannot become action-caused evidence.+ self._status_observation_child_pages: set[Page] = set()+ self._status_observation_capped = False+ # Monotonic deadline for admitting NEW requests into status observation. None until seal, meaning+ # "the action is still running, admit every xhr/fetch"; set at seal to now + grace so requests that+ # start after the grace (unrelated heartbeats during the long download wait) are never observed.+ # Bounds request initiation only: a request admitted before the deadline stays eligible even when+ # its response lands afterward.+ self._status_observation_deadline: float | None = Noneself._child_pages_with_bootstrap_allowance: set[Page] = set()self._drained = asyncio.Event()self._drained.set()@@ -4060,31 +4088,104 @@ class ScopedXhrDownloadCapture:def observed_download_failure_status(self) -> int | None:return self._observed_download_failure_status+ def _within_status_observation_window(self) -> bool:+ deadline = self._status_observation_deadline+ if deadline is None:+ return True+ return self._monotonic() < deadline+def _on_request(self, request: Request) -> None:- redirected_from_admitted_request = request.redirected_from in self._admitted_requestsif not self._active or request.resource_type not in ("xhr", "fetch"):return+ redirected_from_admitted_request = request.redirected_from in self._admitted_requests+child_page_has_bootstrap_allowance = False- if not self._accept_new_requests and request.redirected_from is None:+ bootstrap_provenance_in_window = False+ request_page: Page | None = None+ if request.redirected_from is None:try:request_page = request.frame.page- child_page_has_bootstrap_allowance = request_page in self._child_pages_with_bootstrap_allowanceexcept Exception:- pass+ request_page = None+ if request_page is not None and not self._accept_new_requests:+ child_page_has_bootstrap_allowance = request_page in self._child_pages_with_bootstrap_allowance+ bootstrap_provenance_in_window = request_page in self._status_observation_child_pagesif child_page_has_bootstrap_allowance:self._child_pages_with_bootstrap_allowance.discard(request_page)+ # Status observation is scoped by chain provenance, not the raw deadline. A proven action-owned+ # descendant -- a redirect off an already-OBSERVED request, or a bootstrap hop on a child page+ # opened in-window -- stays eligible even when it begins after the deadline. Wait-extension+ # admission (_admitted_requests) is deliberately broader than observation and is NOT a provenance+ # proxy: a redirect off an admitted-but-unobserved predecessor is excluded. An otherwise+ # unattributed request is eligible only as an in-window root; unrelated new roots at/after the+ # deadline, and redirects off a pre-action/stale/unobserved predecessor, are excluded.+ self._maybe_observe_request(+ request,+ child_page_has_bootstrap_allowance=child_page_has_bootstrap_allowance,+ bootstrap_provenance_in_window=bootstrap_provenance_in_window,+ )++ # One-shot: a child page's status-observation provenance is spent by its FIRST root xhr/fetch --+ # whether that request was observed via the post-deadline bootstrap branch or the general in-window+ # path -- so a later unrelated root from the same child (e.g. a heartbeat after the 2s grace) cannot+ # reuse a stale token and be wrongly attributed. Consumed regardless of the 64 cap (fail closed).+ # The separate wait-extension allowance (_child_pages_with_bootstrap_allowance) is untouched here.+ if request_page is not None:+ self._status_observation_child_pages.discard(request_page)+if self._accept_new_requests or redirected_from_admitted_request or child_page_has_bootstrap_allowance:self._in_flight_requests.add(request)self._admitted_requests.add(request)+ def _is_status_observation_eligible(+ self,+ request: Request,+ *,+ child_page_has_bootstrap_allowance: bool,+ bootstrap_provenance_in_window: bool,+ ) -> bool:+ if request.redirected_from is not None:+ return request.redirected_from in self._status_observation_requests+ if child_page_has_bootstrap_allowance:+ # Bootstrap provenance is additive: it EXTENDS eligibility to a first bootstrap hop past the+ # deadline, but a spent token must not strip an otherwise-ordinary in-grace root of its normal+ # window eligibility.+ return bootstrap_provenance_in_window or self._within_status_observation_window()+ return self._within_status_observation_window()++ def _maybe_observe_request(+ self,+ request: Request,+ *,+ child_page_has_bootstrap_allowance: bool,+ bootstrap_provenance_in_window: bool,+ ) -> None:+ if request in self._status_observation_requests:+ return+ if not self._is_status_observation_eligible(+ request,+ child_page_has_bootstrap_allowance=child_page_has_bootstrap_allowance,+ bootstrap_provenance_in_window=bootstrap_provenance_in_window,+ ):+ return+ if len(self._status_observation_requests) < _MAX_STATUS_OBSERVATION_REQUESTS:+ self._status_observation_requests.add(request)+ elif not self._status_observation_capped:+ self._status_observation_capped = True+ LOG.info(+ "Status-observation request set hit its cap; further requests are not tracked for 5xx observation",+ cap=_MAX_STATUS_OBSERVATION_REQUESTS,+ )+def _on_request_finished(self, request: Request) -> None:self._in_flight_requests.discard(request)def seal_in_flight_requests(self) -> None:self._accept_new_requests = False+ self._status_observation_deadline = self._monotonic() + _STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDSdef _is_xhr_download(self, headers: dict[str, str], status: int) -> bool:"""Check if an XHR response carries a downloadable file body.@@ -4115,15 +4216,17 @@ class ScopedXhrDownloadCapture:task.add_done_callback(self._on_response_done)def _observe_download_failure_status(self, response: Response) -> None:- """Record a server 5xx status for any admitted xhr/fetch request during this download action.+ """Record a server 5xx status for an xhr/fetch request initiated during this download action's window.Runs on Playwright's event loop for every response while the capture is attached, including the- CDP-interceptor lane where response-body capture is off. Stores only the integer status and only- when the request was admitted for this action window. Retains nothing else — no URL, host, or- header — and never raises into the event loop.+ CDP-interceptor lane where response-body capture is off. Observation is scoped by window+ membership (_status_observation_requests), NOT by wait-extension admission: a JS-initiated second+ hop that starts after seal is still observed, while a request already in flight before this action+ (never seen by _on_request) is excluded so it cannot become action-caused evidence. Stores only+ the integer status — no URL, host, or header — and never raises into the event loop."""try:- if response.request not in self._admitted_requests:+ if response.request not in self._status_observation_requests:returnstatus = response.statusif not isinstance(status, int) or status not in _OBSERVED_DOWNLOAD_FAILURE_STATUSES:@@ -4220,6 +4323,8 @@ class ScopedXhrDownloadCapture:if not self._active:returnself._child_pages_with_bootstrap_allowance.add(page)+ if self._within_status_observation_window():+ self._status_observation_child_pages.add(page)self._attach_page(page)self._extra_pages.append(page)@@ -4259,6 +4364,9 @@ class ScopedXhrDownloadCapture:self._extra_pages.clear()self._child_pages_with_bootstrap_allowance.clear()self._in_flight_requests.clear()+ self._status_observation_requests.clear()+ self._status_observation_child_pages.clear()+ self._status_observation_deadline = Noneclass ActionHandler:diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.pyindex 5c9cfc747..84434cb3c 100644--- a/tests/unit/test_browser_effect_sink_enumeration.py+++ b/tests/unit/test_browser_effect_sink_enumeration.py@@ -22,7 +22,7 @@ _DISCOVERED_BROWSER_API_CALLS = {"bring_to_front": 2,"check": 2,"click": 24,- "clear": 4,+ "clear": 6,# 6 = 5 prior + the bounded v4 FileDownloadBlock synchronous popup close (SKY-15371)."close": 6,"dblclick": 2,@@ -137,6 +137,8 @@ _NON_BROWSER_CANDIDATES = Counter(("disable", "clear", "self._child_pages_with_bootstrap_allowance"): 1,("disable", "clear", "self._extra_pages"): 1,("disable", "clear", "self._in_flight_requests"): 1,+ ("disable", "clear", "self._status_observation_requests"): 1,+ ("disable", "clear", "self._status_observation_child_pages"): 1,})@@ -225,13 +227,13 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:}assert observed == _DISCOVERED_BROWSER_API_CALLS- assert sum(sum(methods.values()) for methods in observed.values()) == 171+ assert sum(sum(methods.values()) for methods in observed.values()) == 173handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)classified_non_browser = Counter({signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES})assert classified_non_browser == _NON_BROWSER_CANDIDATES- assert sum(_NON_BROWSER_CANDIDATES.values()) == 5+ assert sum(_NON_BROWSER_CANDIDATES.values()) == 7assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 166diff --git a/tests/unit/test_scoped_xhr_download_capture.py b/tests/unit/test_scoped_xhr_download_capture.pyindex 6553c34c9..8891a829b 100644--- a/tests/unit/test_scoped_xhr_download_capture.py+++ b/tests/unit/test_scoped_xhr_download_capture.py@@ -7,7 +7,24 @@ from unittest.mock import AsyncMock, MagicMock, PropertyMockimport pytestfrom structlog.testing import capture_logs-from skyvern.webeye.actions.handler import ScopedXhrDownloadCapture+from skyvern.webeye.actions.handler import (+ _MAX_STATUS_OBSERVATION_REQUESTS,+ _STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS,+ ScopedXhrDownloadCapture,+)+++class _FakeClock:+ """Deterministic monotonic clock driven by the test, avoiding real sleeps."""++ def __init__(self, now: float = 0.0) -> None:+ self._now = now++ def __call__(self) -> float:+ return self._now++ def advance(self, seconds: float) -> None:+ self._now += secondsdef _make_response(@@ -857,3 +874,397 @@ class TestScopedXhrDownloadFailureStatusObserver:serialized = repr(vars(capture))assert _ADMITTED_URL not in serializedassert _ADMITTED_HOST_CANARY not in serialized+++class TestScopedXhrStatusObservationBounds:+ """Boundary/cleanup for the passive status-observation set (SKY-9971 _MAX_STATUS_OBSERVATION_REQUESTS)."""++ @staticmethod+ def _response_for(request: MagicMock, status: int) -> MagicMock:+ resp = MagicMock()+ resp.request = request+ resp.status = status+ return resp++ def test_observation_set_caps_at_max_and_excludes_overflow(self) -> None:+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"))+ capture.enable()+ reqs = [_make_request() for _ in range(_MAX_STATUS_OBSERVATION_REQUESTS + 1)]+ for req in reqs:+ capture._on_request(req)++ assert len(capture._status_observation_requests) == _MAX_STATUS_OBSERVATION_REQUESTS+ assert capture._status_observation_capped is True+ # Last request at the cap retained, the one past it excluded.+ assert reqs[_MAX_STATUS_OBSERVATION_REQUESTS - 1] in capture._status_observation_requests+ assert reqs[_MAX_STATUS_OBSERVATION_REQUESTS] not in capture._status_observation_requests++ capture._observe_download_failure_status(self._response_for(reqs[_MAX_STATUS_OBSERVATION_REQUESTS], 500))+ assert capture.observed_download_failure_status is None++ capture._observe_download_failure_status(self._response_for(reqs[0], 503))+ assert capture.observed_download_failure_status == 503++ def test_cap_is_64_and_excludes_65th_eligible_request(self) -> None:+ # Concrete-value guard on the defensive bound: filling 65 in-window-eligible requests must retain+ # exactly 64 and drop the 65th, so a later request cannot stamp status while an earlier retained+ # one still can. Fails at the prior cap of 512 (all 65 retained, never capped).+ assert _MAX_STATUS_OBSERVATION_REQUESTS == 64+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"))+ capture.enable()+ reqs = [_make_request() for _ in range(65)]+ for req in reqs:+ capture._on_request(req)++ assert len(capture._status_observation_requests) == 64+ assert capture._status_observation_capped is True+ assert reqs[63] in capture._status_observation_requests+ assert reqs[64] not in capture._status_observation_requests++ capture._observe_download_failure_status(self._response_for(reqs[64], 500))+ assert capture.observed_download_failure_status is None++ capture._observe_download_failure_status(self._response_for(reqs[0], 503))+ assert capture.observed_download_failure_status == 503++ def test_over_cap_late_request_neither_extends_wait_nor_is_observed(self) -> None:+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"))+ capture.enable()+ # Fill to the cap and finish each so none linger in the wait-extension set.+ for _ in range(_MAX_STATUS_OBSERVATION_REQUESTS):+ req = _make_request()+ capture._on_request(req)+ capture._on_request_finished(req)+ assert capture.has_in_flight_requests is False++ capture.seal_in_flight_requests()+ late = _make_request()+ capture._on_request(late)++ assert late not in capture._status_observation_requests # over cap -> excluded from observation+ assert late not in capture._admitted_requests # post-seal -> never admitted+ assert capture.has_in_flight_requests is False # never enters wait-extension state++ capture._observe_download_failure_status(self._response_for(late, 500))+ assert capture.observed_download_failure_status is None++ def test_post_seal_in_window_request_is_observed_but_not_wait_extending(self) -> None:+ # Load-bearing: a request initiated after seal (the delayed second hop) is still observed for+ # status yet never joins the wait-extension admission set.+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"))+ capture.enable()+ capture.seal_in_flight_requests()++ late = _make_request()+ capture._on_request(late)+ assert late in capture._status_observation_requests+ assert late not in capture._admitted_requests+ assert capture.has_in_flight_requests is False++ capture._observe_download_failure_status(self._response_for(late, 502))+ assert capture.observed_download_failure_status == 502++ def test_disable_clears_retained_status_observation_references(self) -> None:+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"))+ capture.enable()+ for _ in range(10):+ capture._on_request(_make_request())+ assert capture._status_observation_requests+ assert capture.has_in_flight_requests is True++ capture.disable()+ assert capture._status_observation_requests == set()+ assert capture._in_flight_requests == set()+ assert capture._active is False+++class TestScopedXhrStatusObservationPostSealGrace:+ """The post-seal grace window bounds when a NEW request may enter status observation (SKY-9971).++ Without it, ``_active`` stays true across the whole 240-600s download wait, so an unrelated 5xx+ heartbeat fired minutes after the action could stamp ``download_failure_status``.+ """++ @staticmethod+ def _response_for(request: MagicMock, status: int) -> MagicMock:+ resp = MagicMock()+ resp.request = request+ resp.status = status+ return resp++ def test_request_within_grace_is_observed_even_if_response_lands_after_deadline(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()+ capture.seal_in_flight_requests() # deadline = 0 + grace++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS / 2) # still inside the grace+ second_hop = _make_request()+ capture._on_request(second_hop)+ assert second_hop in capture._status_observation_requests++ # Deadline bounds initiation, not completion: the response arrives well after the deadline.+ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 100)+ capture._observe_download_failure_status(self._response_for(second_hop, 503))+ assert capture.observed_download_failure_status == 503++ def test_request_after_grace_is_not_observed_and_5xx_does_not_stamp(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS + 0.01) # just past the deadline+ heartbeat = _make_request()+ capture._on_request(heartbeat)+ assert heartbeat not in capture._status_observation_requests++ capture._observe_download_failure_status(self._response_for(heartbeat, 503))+ assert capture.observed_download_failure_status is None++ def test_request_at_deadline_is_not_observed(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS) # monotonic() == deadline -> excluded+ at_deadline = _make_request()+ capture._on_request(at_deadline)+ assert at_deadline not in capture._status_observation_requests++ def test_pre_seal_requests_observed_regardless_of_clock(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()++ clock.advance(9999) # no deadline before seal: the whole action window is open+ during_action = _make_request()+ capture._on_request(during_action)+ assert during_action in capture._status_observation_requests++ def test_disable_resets_post_seal_deadline(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()+ capture.seal_in_flight_requests()+ assert capture._status_observation_deadline is not None++ capture.disable()+ assert capture._status_observation_deadline is None+++class TestScopedXhrStatusObservationChainProvenance:+ """Chain provenance for passive status observation (SKY-9971 review r3982881754 / r3983407333).++ The post-seal deadline bounds where a NEW ROOT may enter observation, but a request already proven+ action-owned inside the window -- a redirect off an admitted/observed request, or a bootstrap hop on a+ child page opened in-window -- must stay eligible when its next hop begins after the deadline. A+ redirect off a pre-action/stale root, whose predecessor was never admitted or observed, stays excluded+ even inside the grace.+ """++ @staticmethod+ def _response_for(request: MagicMock, status: int) -> MagicMock:+ resp = MagicMock()+ resp.request = request+ resp.status = status+ return resp++ def test_in_window_admitted_root_redirect_after_deadline_stays_observed(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()+ root = _make_request()+ capture._on_request(root) # admitted + observed while the action is live+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 5) # well past the deadline+ redirect = _make_request(redirected_from=root)+ capture._on_request(redirect)++ assert redirect in capture._status_observation_requests+ capture._observe_download_failure_status(self._response_for(redirect, 503))+ assert capture.observed_download_failure_status == 503++ def test_pre_capture_root_redirect_during_grace_is_excluded(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS / 2) # inside the grace+ pre_capture_root = _make_request() # deliberately NOT fed to _on_request: it began before capture+ redirect = _make_request(redirected_from=pre_capture_root)+ capture._on_request(redirect)++ assert redirect not in capture._status_observation_requests+ capture._observe_download_failure_status(self._response_for(redirect, 500))+ assert capture.observed_download_failure_status is None++ def test_new_root_after_deadline_excluded_even_with_proven_sibling_chain(self) -> None:+ # Guard: an entirely new JS-triggered root after the fixed deadline is excluded even though an+ # unrelated admitted chain exists. Proven-owned eligibility never leaks to unrelated new roots.+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()+ capture._on_request(_make_request()) # an admitted chain the new root is not part of+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 5)+ new_root = _make_request()+ capture._on_request(new_root)++ assert new_root not in capture._status_observation_requests+ capture._observe_download_failure_status(self._response_for(new_root, 502))+ assert capture.observed_download_failure_status is None++ def test_redirect_off_post_seal_observed_root_stays_eligible(self) -> None:+ # A JS second hop root started in-grace is observed but not admitted; its own redirect after the+ # deadline must stay eligible via the OBSERVATION chain, not merely the admitted chain.+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ capture.enable()+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS / 2)+ second_hop_root = _make_request()+ capture._on_request(second_hop_root)+ assert second_hop_root in capture._status_observation_requests+ assert second_hop_root not in capture._admitted_requests++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 5)+ redirect = _make_request(redirected_from=second_hop_root)+ capture._on_request(redirect)++ assert redirect in capture._status_observation_requests+ capture._observe_download_failure_status(self._response_for(redirect, 504))+ assert capture.observed_download_failure_status == 504++ def test_child_bootstrap_after_deadline_observed_when_page_opened_in_window(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ child_page = _make_page()+ capture.enable()+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS / 2) # child page opens in-window+ capture._on_new_page(child_page)++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 5) # bootstrap hop fires after deadline+ child_request = _make_request(page=child_page)+ capture._on_request(child_request)++ assert child_request in capture._status_observation_requests+ capture._observe_download_failure_status(self._response_for(child_request, 500))+ assert capture.observed_download_failure_status == 500++ def test_child_bootstrap_excluded_when_page_opened_after_deadline(self) -> None:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ child_page = _make_page()+ capture.enable()+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 5) # child page opens AFTER the deadline+ capture._on_new_page(child_page)+ child_request = _make_request(page=child_page)+ capture._on_request(child_request)++ # Still admitted for wait extension (unchanged), but never observed: its provenance is out-of-window.+ assert child_request in capture._admitted_requests+ assert child_request not in capture._status_observation_requests+ capture._observe_download_failure_status(self._response_for(child_request, 500))+ assert capture.observed_download_failure_status is None++ def test_redirect_off_out_of_window_child_bootstrap_is_excluded(self) -> None:+ # A child page opened AFTER the deadline yields a bootstrap request that is admitted for wait+ # extension but correctly excluded from observation. A redirect off that admitted-but-unobserved+ # request must NOT re-enter observation through the broader wait-admission set: redirect eligibility+ # requires the predecessor's OWN observation provenance, not mere admission.+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ child_page = _make_page()+ capture.enable()+ capture.seal_in_flight_requests()++ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 5) # child page opens AFTER the deadline+ capture._on_new_page(child_page)+ child_request = _make_request(page=child_page)+ capture._on_request(child_request)+ assert child_request in capture._admitted_requests+ assert child_request not in capture._status_observation_requests++ redirect = _make_request(redirected_from=child_request)+ capture._on_request(redirect)++ # Predecessor was admitted-but-not-observed -> the redirect stays out of observation and cannot stamp.+ assert redirect not in capture._status_observation_requests+ capture._observe_download_failure_status(self._response_for(redirect, 500))+ assert capture.observed_download_failure_status is None+ # Normal wait extension is preserved: a redirect off an admitted request is still tracked.+ assert redirect in capture._admitted_requests++ def test_child_first_in_window_request_consumes_status_provenance(self) -> None:+ # A child page opened in-window whose FIRST xhr/fetch fires while the action is still live+ # (pre-seal) is observed via the general in-window path; its one-shot status-observation provenance+ # must be spent then. Otherwise a later unrelated root from the same child after the 2s grace (e.g.+ # a heartbeat during the long download wait) re-consumes the stale token and is wrongly stamped.+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ child_page = _make_page()+ capture.enable()+ capture._on_new_page(child_page) # child opens in-window (pre-seal)++ first = _make_request(page=child_page)+ capture._on_request(first) # first hop fires BEFORE seal -> observed via the in-window path+ assert first in capture._status_observation_requests+ assert child_page not in capture._status_observation_child_pages # provenance consumed++ capture.seal_in_flight_requests()+ clock.advance(_STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 5) # past the deadline++ later = _make_request(page=child_page)+ capture._on_request(later) # unrelated later root from the same child, after the grace+ assert later not in capture._status_observation_requests+ capture._observe_download_failure_status(self._response_for(later, 500))+ assert capture.observed_download_failure_status is None+ # Wait-extension allowance is separate and unchanged: the later child root is still admitted.+ assert later in capture._admitted_requests++ def _observe_child_root(self, *, pre_seal_root: bool, advance: float) -> bool:+ clock = _FakeClock()+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"), monotonic=clock)+ child_page = _make_page()+ capture.enable()+ capture._on_new_page(child_page) # child opened in-window+ if pre_seal_root:+ capture._on_request(_make_request(page=child_page)) # first root pre-seal spends the status token+ capture.seal_in_flight_requests()+ clock.advance(advance)+ req = _make_request(page=child_page)+ capture._on_request(req)+ return req in capture._status_observation_requests++ def test_in_grace_child_root_eligibility_matrix(self) -> None:+ # Bootstrap provenance is ADDITIVE to ordinary in-grace eligibility, not an override. An in-grace+ # (+0.5s) child root is observed whether or not the child made an earlier root; an after-grace+ # (+10s) root is observed only via an UNSPENT bootstrap token, and a spent token blocks the later+ # heartbeat.+ in_grace = _STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS / 2+ after_grace = _STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS * 5++ assert self._observe_child_root(pre_seal_root=False, advance=in_grace) is True+ assert self._observe_child_root(pre_seal_root=True, advance=in_grace) is True # regression case+ assert self._observe_child_root(pre_seal_root=False, advance=after_grace) is True+ assert self._observe_child_root(pre_seal_root=True, advance=after_grace) is False++ def test_disable_clears_status_observation_child_page_provenance(self) -> None:+ capture = ScopedXhrDownloadCapture(_make_page(), Path("/tmp/downloads"))+ child_page = _make_page()+ capture.enable()+ capture._on_new_page(child_page)+ assert capture._status_observation_child_pages++ capture.disable()+ assert capture._status_observation_child_pages == set()