Stealth browser automation for AI agents over MCP and the Chrome DevTools Protocol: navigation, network hooks, DOM extraction, and pixel-accurate UI cloning.
Python ai-agent-toolsai-toolsanthropicanti-bot-bypassautomation-toolsbrowser-automationchrome-devtoolsclaude-mcpcloudflare-bypassdeveloper-toolsfastmcpmcp-servermodel-context-protocolnetwork-interceptionnodriverpythonstealth-browserundetectable-browserweb-automationweb-scraping
fix(navigation): recover stale tabs during long runs
Vibhek Soni6fdcf5ce3e8d
diff --git a/src/browser_manager.py b/src/browser_manager.pyindex e9485c6..112f78a 100644--- a/src/browser_manager.py+++ b/src/browser_manager.py@@ -2,6 +2,7 @@import asyncioimport sys+import timeimport uuidfrom typing import Any, Dict, Optional, Listfrom datetime import datetime, timedelta@@ -28,6 +29,8 @@ from proxy_utils import (class BrowserManager:"""Manages multiple browser instances."""+ NAVIGATION_RECYCLE_THRESHOLD = 25+def __init__(self):self._instances: Dict[str, dict] = {}self._lock = asyncio.Lock()@@ -218,6 +221,7 @@ class BrowserManager:'tab': tab,'instance': instance,'options': options,+ 'navigation_count': 0,'spawn_diagnostics': spawn_diagnostics,'network_data': []}@@ -439,6 +443,271 @@ class BrowserManager:"""Get spawn diagnostics for an instance."""return self._spawn_diagnostics.get(instance_id)+ @staticmethod+ def _get_tab_target_id(tab: Optional[Tab]) -> Optional[str]:+ """Get a stable target id string for a tab when available."""+ if tab is None:+ return None+ target = getattr(tab, "target", None)+ target_id = getattr(target, "target_id", None)+ if target_id is None:+ return None+ return str(target_id)++ @staticmethod+ def _is_recoverable_navigation_error(error: Exception) -> bool:+ """Return whether a navigation error should trigger one stale-tab recovery attempt."""+ if isinstance(error, asyncio.TimeoutError):+ return True++ message = f"{type(error).__name__}: {error}".lower()+ recoverable_markers = (+ "connection dropped",+ "connection closed",+ "connection lost",+ "websocket",+ "target closed",+ "target crashed",+ "session closed",+ "invalid state",+ "not attached",+ )+ return any(marker in message for marker in recoverable_markers)++ async def _replace_main_tab(+ self,+ instance_id: str,+ reason: str,+ close_existing: bool = True,+ ) -> Optional[Tab]:+ """+ Replace the tracked main tab for an instance with a fresh about:blank tab.++ Args:+ instance_id (str): Browser instance id.+ reason (str): Diagnostic reason for replacement.+ close_existing (bool): Whether to close the previously tracked tab.++ Returns:+ Optional[Tab]: The fresh tab, or None if the instance was missing.+ """+ data = await self.get_instance(instance_id)+ if not data:+ return None++ browser = data["browser"]+ previous_tab = data.get("tab")+ new_tab = await browser.get("about:blank", new_tab=True)+ await new_tab++ if close_existing and previous_tab:+ previous_target_id = self._get_tab_target_id(previous_tab)+ new_target_id = self._get_tab_target_id(new_tab)+ if previous_target_id and previous_target_id != new_target_id:+ try:+ await previous_tab.close()+ except Exception:+ pass++ async with self._lock:+ if instance_id in self._instances:+ self._instances[instance_id]["tab"] = new_tab+ self._instances[instance_id]["navigation_count"] = 0++ debug_logger.log_info(+ "browser_manager",+ "_replace_main_tab",+ f"Replaced main tab for {instance_id}: {reason}",+ )+ return new_tab++ async def get_navigation_tab(self, instance_id: str) -> Optional[Tab]:+ """+ Get a healthy tab for navigation, recovering from stale tracked tabs when needed.++ Args:+ instance_id (str): Browser instance id.++ Returns:+ Optional[Tab]: A valid navigation tab, or None if the instance does not exist.+ """+ data = await self.get_instance(instance_id)+ if not data:+ return None++ browser = data["browser"]+ tracked_tab = data.get("tab")+ navigation_count = data.get("navigation_count", 0)++ if (+ self.NAVIGATION_RECYCLE_THRESHOLD > 0+ and navigation_count >= self.NAVIGATION_RECYCLE_THRESHOLD+ ):+ return await self._replace_main_tab(+ instance_id,+ reason=f"navigation recycle threshold {self.NAVIGATION_RECYCLE_THRESHOLD} reached",+ )++ try:+ await browser.update_targets()+ tracked_target_id = self._get_tab_target_id(tracked_tab)+ if tracked_target_id:+ for candidate_tab in browser.tabs:+ if self._get_tab_target_id(candidate_tab) == tracked_target_id:+ await candidate_tab+ return candidate_tab++ if browser.tabs:+ fallback_tab = browser.tabs[0]+ await fallback_tab+ async with self._lock:+ if instance_id in self._instances:+ self._instances[instance_id]["tab"] = fallback_tab+ return fallback_tab+ except Exception as error:+ debug_logger.log_warning(+ "browser_manager",+ "get_navigation_tab",+ f"Tab health check failed for {instance_id}: {error}",+ )++ return await self._replace_main_tab(+ instance_id,+ reason="tracked tab missing or invalid",+ close_existing=False,+ )++ @staticmethod+ async def _wait_for_navigation_condition(+ tab: Tab,+ wait_until: str,+ timeout_seconds: float,+ ) -> None:+ """+ Wait for a navigation milestone within the remaining timeout budget.++ Args:+ tab (Tab): Browser tab.+ wait_until (str): Desired wait condition.+ timeout_seconds (float): Remaining timeout budget in seconds.+ """+ if timeout_seconds <= 0:+ raise asyncio.TimeoutError("Navigation wait budget exhausted")++ if wait_until == "domcontentloaded":+ await asyncio.wait_for(+ tab.wait(uc.cdp.page.DomContentEventFired),+ timeout=timeout_seconds,+ )+ return++ if wait_until == "networkidle":+ await asyncio.sleep(min(timeout_seconds, 2.0))+ return++ await asyncio.wait_for(+ tab.wait(uc.cdp.page.LoadEventFired),+ timeout=timeout_seconds,+ )++ async def navigate(+ self,+ instance_id: str,+ url: str,+ wait_until: str = "load",+ timeout: int = 30000,+ referrer: Optional[str] = None,+ ) -> Dict[str, Any]:+ """+ Navigate with timeout enforcement and one automatic tab-recovery retry.++ Args:+ instance_id (str): Browser instance id.+ url (str): Target URL.+ wait_until (str): Wait condition after navigation.+ timeout (int): Timeout in milliseconds.+ referrer (Optional[str]): Optional referrer header.++ Returns:+ Dict[str, Any]: Navigation result payload.+ """+ timeout_seconds = max(timeout, 1) / 1000+ last_error: Optional[Exception] = None++ for attempt in range(2):+ if attempt == 0:+ tab = await self.get_navigation_tab(instance_id)+ else:+ tab = await self._replace_main_tab(+ instance_id,+ reason=f"recovering after navigation failure: {type(last_error).__name__ if last_error else 'unknown'}",+ )++ if not tab:+ raise Exception(f"Instance not found: {instance_id}")++ start_time = time.monotonic()++ try:+ if referrer:+ await tab.send(+ uc.cdp.network.set_extra_http_headers(+ headers={"Referer": referrer}+ )+ )++ await asyncio.wait_for(tab.get(url), timeout=timeout_seconds)++ elapsed = time.monotonic() - start_time+ await self._wait_for_navigation_condition(+ tab,+ wait_until,+ timeout_seconds - elapsed,+ )++ elapsed = time.monotonic() - start_time+ remaining = timeout_seconds - elapsed+ if remaining <= 0:+ raise asyncio.TimeoutError("Navigation result budget exhausted")++ final_url = await asyncio.wait_for(+ tab.evaluate("window.location.href"),+ timeout=remaining,+ )+ title = await asyncio.wait_for(+ tab.evaluate("document.title"),+ timeout=remaining,+ )++ await self.update_instance_state(instance_id, final_url, title)++ async with self._lock:+ if instance_id in self._instances:+ self._instances[instance_id]["tab"] = tab+ self._instances[instance_id]["navigation_count"] = (+ self._instances[instance_id].get("navigation_count", 0) + 1+ )++ return {+ "url": final_url,+ "title": title,+ "success": True,+ }+ except Exception as error:+ last_error = error+ debug_logger.log_warning(+ "browser_manager",+ "navigate",+ f"Navigation attempt {attempt + 1} failed for {instance_id}: {error}",+ {"url": url, "attempt": attempt + 1},+ )+ if attempt == 1 or not self._is_recoverable_navigation_error(error):+ if isinstance(error, asyncio.TimeoutError):+ raise Exception(+ f"Navigation to {url} timed out after {timeout}ms"+ ) from error+ raise+async def get_tab(self, instance_id: str) -> Optional[Tab]:"""Get the main tab for a browser instance.diff --git a/src/server.py b/src/server.pyindex 1adcf71..2b4b6ca 100644--- a/src/server.py+++ b/src/server.py@@ -295,31 +295,13 @@ async def navigate("""if isinstance(timeout, str):timeout = int(timeout)- tab = await browser_manager.get_tab(instance_id)- if not tab:- raise Exception(f"Instance not found: {instance_id}")- try:- if referrer:- await tab.send(uc.cdp.network.set_extra_http_headers(- headers={"Referer": referrer}- ))- await tab.get(url)- if wait_until == "domcontentloaded":- await tab.wait(uc.cdp.page.DomContentEventFired)- elif wait_until == "networkidle":- await asyncio.sleep(2)- else:- await tab.wait(uc.cdp.page.LoadEventFired)- final_url = await tab.evaluate("window.location.href")- title = await tab.evaluate("document.title")- await browser_manager.update_instance_state(instance_id, final_url, title)- return {- "url": final_url,- "title": title,- "success": True- }- except Exception as e:- raise+ return await browser_manager.navigate(+ instance_id=instance_id,+ url=url,+ wait_until=wait_until,+ timeout=timeout,+ referrer=referrer,+ )@section_tool("browser-management")async def go_back(instance_id: str) -> bool: