Skip to content

The agent that turns websites into APIs!

18.9 KB
Raw Blame
"""Browser management with Playwright for HAR recording."""

import io
import json
import os
import random
import signal
import sys
from collections.abc import Callable
from pathlib import Path
from types import FrameType

from playwright.sync_api import Browser, BrowserContext, Page, Playwright, sync_playwright
from playwright_stealth import Stealth
from rich.console import Console
from rich.status import Status

from .utils import get_har_dir, get_timestamp

console = Console()

# Null stderr stream for suppressing logs
_null_stderr = io.StringIO()


def _null_logger(message: dict) -> None:
    """Null logger that discards all messages."""
    pass


# Realistic Chrome user agents (updated for late 2024/2025)
USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
]

# Stealth JavaScript to inject - bypasses common detection methods
STEALTH_JS = """
// Override navigator.webdriver
Object.defineProperty(navigator, 'webdriver', {
    get: () => undefined,
});

// Override navigator.plugins to look like a real browser
Object.defineProperty(navigator, 'plugins', {
    get: () => {
        const plugins = [
            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
        ];
        plugins.item = (index) => plugins[index];
        plugins.namedItem = (name) => plugins.find(p => p.name === name) || null;
        plugins.refresh = () => {};
        return plugins;
    },
});

// Override navigator.languages
Object.defineProperty(navigator, 'languages', {
    get: () => ['en-US', 'en'],
});

// Override navigator.permissions.query for notifications
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => {
    if (parameters.name === 'notifications') {
        return Promise.resolve({ state: Notification.permission });
    }
    return originalQuery(parameters);
};

// Remove automation-related properties from window
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Object;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Proxy;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;

// Override chrome runtime to look authentic
if (!window.chrome) {
    window.chrome = {};
}
window.chrome.runtime = {
    PlatformOs: { MAC: 'mac', WIN: 'win', ANDROID: 'android', CROS: 'cros', LINUX: 'linux', OPENBSD: 'openbsd' },
    PlatformArch: { ARM: 'arm', X86_32: 'x86-32', X86_64: 'x86-64' },
    PlatformNaclArch: { ARM: 'arm', X86_32: 'x86-32', X86_64: 'x86-64' },
    RequestUpdateCheckStatus: { THROTTLED: 'throttled', NO_UPDATE: 'no_update', UPDATE_AVAILABLE: 'update_available' },
    OnInstalledReason: { INSTALL: 'install', UPDATE: 'update', CHROME_UPDATE: 'chrome_update', SHARED_MODULE_UPDATE: 'shared_module_update' },
    OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
};

// Fix iframe contentWindow detection
const originalAttachShadow = Element.prototype.attachShadow;
Element.prototype.attachShadow = function(init) {
    if (init && init.mode === 'closed') {
        init.mode = 'open';
    }
    return originalAttachShadow.call(this, init);
};

// Override WebGL vendor/renderer to look consistent
const getParameter = WebGLRenderingContext.prototype.getParameter;
WebGLRenderingContext.prototype.getParameter = function(parameter) {
    if (parameter === 37445) { // UNMASKED_VENDOR_WEBGL
        return 'Google Inc. (Apple)';
    }
    if (parameter === 37446) { // UNMASKED_RENDERER_WEBGL
        return 'ANGLE (Apple, ANGLE Metal Renderer: Apple M1 Pro, Unspecified Version)';
    }
    return getParameter.call(this, parameter);
};

// Do the same for WebGL2
const getParameter2 = WebGL2RenderingContext.prototype.getParameter;
WebGL2RenderingContext.prototype.getParameter = function(parameter) {
    if (parameter === 37445) {
        return 'Google Inc. (Apple)';
    }
    if (parameter === 37446) {
        return 'ANGLE (Apple, ANGLE Metal Renderer: Apple M1 Pro, Unspecified Version)';
    }
    return getParameter2.call(this, parameter);
};

// Override Permissions API
const originalPermissionsQuery = navigator.permissions.query;
navigator.permissions.query = function(permissionDesc) {
    if (permissionDesc.name === 'notifications') {
        return Promise.resolve({
            state: 'prompt',
            onchange: null
        });
    }
    return originalPermissionsQuery.call(navigator.permissions, permissionDesc);
};

// Spoof hardwareConcurrency to a realistic value
Object.defineProperty(navigator, 'hardwareConcurrency', {
    get: () => 8,
});

// Spoof deviceMemory
Object.defineProperty(navigator, 'deviceMemory', {
    get: () => 8,
});

// Spoof connection info
if (navigator.connection) {
    Object.defineProperty(navigator.connection, 'rtt', {
        get: () => 50,
    });
}

// Hide automation in chrome.app
if (window.chrome && window.chrome.app) {
    window.chrome.app.isInstalled = false;
    window.chrome.app.InstallState = { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' };
    window.chrome.app.RunningState = { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' };
}

console.log('Stealth mode activated');
"""


def get_chrome_profile_dir() -> Path | None:
    """Get Chrome's platform-specific user data directory if it exists."""
    if sys.platform == "win32":
        root = Path(os.environ.get("LOCALAPPDATA") or Path.home() / "AppData/Local")
        profile = root / "Google/Chrome/User Data"
    elif sys.platform == "darwin":
        profile = Path.home() / "Library/Application Support/Google/Chrome"
    else:
        root = Path(os.environ.get("CHROME_CONFIG_HOME") or os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
        profile = root / "google-chrome"
    if profile.is_dir():
        return profile
    return None


class ManualBrowser:
    """Manages a Playwright browser session with HAR recording.

    Supports two modes:
    - Real Chrome: Uses your actual Chrome browser with existing profile (best for stealth)
    - Stealth Chromium: Falls back to Playwright's Chromium with stealth patches
    """

    def __init__(
        self,
        run_id: str,
        prompt: str,
        output_dir: str | None = None,
        use_real_chrome: bool = True,  # New option to use real Chrome
    ) -> None:
        self.run_id = run_id
        self.prompt = prompt
        self.output_dir = output_dir
        self.use_real_chrome = use_real_chrome

        self.har_dir = get_har_dir(run_id, output_dir)
        self.har_path = self.har_dir / "recording.har"
        self.metadata_path = self.har_dir / "metadata.json"

        self._playwright: Playwright | None = None
        self._browser: Browser | None = None
        self._context: BrowserContext | None = None
        self._start_time: str | None = None
        self._user_agent = random.choice(USER_AGENTS)
        self._using_persistent = False  # Track if using persistent context

    def _save_metadata(self, end_time: str) -> None:
        """Save run metadata to JSON file."""
        metadata = {
            "run_id": self.run_id,
            "prompt": self.prompt,
            "start_time": self._start_time,
            "end_time": end_time,
            "har_file": str(self.har_path),
        }
        with open(self.metadata_path, "w", encoding="utf-8") as f:
            json.dump(metadata, f, indent=2)

    def _handle_signal(self, signum: int, frame: FrameType | None) -> None:
        """Handle interrupt signals gracefully."""
        console.print("\n\n [dim]terminating capture...[/dim]")
        self.close()
        sys.exit(0)

    def _inject_stealth(self, page: Page) -> None:
        """Inject stealth scripts into page before any other scripts run."""
        page.add_init_script(STEALTH_JS)

    @staticmethod
    def _normalize_url(url: str) -> str:
        """Prepend https:// when the user typed a bare host (e.g.
        ``jobs.example.com/x``); Playwright's goto rejects scheme-less URLs."""
        url = url.strip()
        if url and "://" not in url:
            return "https://" + url
        return url

    def _abort_playwright(self) -> None:
        """Quietly tear down the browser/Playwright after a failed start.

        Unlike close(), this saves nothing and prints nothing — its only job is
        to stop the sync-Playwright event loop so it doesn't leak into the
        caller. A leaked loop makes the caller's next asyncio.run() (the REPL
        prompt) fail with "asyncio.run() cannot be called from a running event
        loop". Safe to call with partially-initialized state.
        """
        teardowns: list[Callable[[], None]] = []
        if self._context is not None:
            teardowns.append(self._context.close)
        if self._browser is not None and not self._using_persistent:
            teardowns.append(self._browser.close)
        if self._playwright is not None:
            teardowns.append(self._playwright.stop)
        for teardown in teardowns:
            try:
                teardown()
            except Exception:
                pass
        self._context = None
        self._browser = None
        self._playwright = None

    def _require_playwright(self) -> Playwright:
        if self._playwright is None:
            raise RuntimeError("Playwright has not been started")
        return self._playwright

    def _start_with_real_chrome(self, start_url: str | None = None) -> Path:
        """Start using the real Chrome browser with user's profile."""
        import shutil
        import tempfile

        chrome_profile = get_chrome_profile_dir()
        if not chrome_profile:
            console.print(" [yellow]chrome profile not found, falling back to stealth mode[/yellow]")
            return self._start_with_stealth_chromium(start_url)

        # Create a temporary profile directory
        temp_profile_dir = Path(tempfile.mkdtemp(prefix="chrome_profile_"))

        console.print(" [dim]using real chrome (profile copy)[/dim]")
        console.print(" [yellow]⚠️  please browse in the FIRST tab only[/yellow]")
        console.print(" [yellow]    (new tabs may not be recorded)[/yellow]")
        console.print()

        try:
            # Use launch_persistent_context with channel="chrome" to use real Chrome binary
            self._context = self._require_playwright().chromium.launch_persistent_context(
                user_data_dir=str(temp_profile_dir),
                channel="chrome",  # Use real Chrome binary
                headless=False,
                record_har_path=str(self.har_path),
                record_har_content="embed",
                no_viewport=True,
                args=[
                    "--start-maximized",
                    "--disable-blink-features=AutomationControlled",
                ],
                ignore_default_args=["--enable-automation", "--no-sandbox"],
            )
            self._using_persistent = True

            for existing_page in self._context.pages:
                try:
                    existing_page.close()
                except Exception:
                    pass

            # For HAR recording & context
            page = self._context.new_page()

            if start_url:
                page.goto(start_url, wait_until="domcontentloaded")
            else:
                page.goto("https://www.google.com", wait_until="domcontentloaded")

            # Wait for browser to close
            try:
                while self._context.pages:
                    self._context.pages[0].wait_for_timeout(100)
            except Exception:
                pass

            return self.close()

        finally:
            # Clean up temp profile
            try:
                shutil.rmtree(temp_profile_dir, ignore_errors=True)
            except Exception:
                pass

    def _start_with_stealth_chromium(self, start_url: str | None = None) -> Path:
        """Start using Playwright's Chromium with stealth patches."""
        # Comprehensive stealth Chrome arguments
        chrome_args = [
            "--start-maximized",
            "--disable-blink-features=AutomationControlled",
            "--disable-features=IsolateOrigins,site-per-process",
            "--disable-infobars",
            "--disable-dev-shm-usage",
            "--disable-background-networking",
            "--disable-default-apps",
            "--disable-extensions",
            "--disable-sync",
            "--disable-translate",
            "--no-first-run",
            "--no-default-browser-check",
            "--no-service-autorun",
            "--disable-backgrounding-occluded-windows",
            "--disable-renderer-backgrounding",
            "--disable-background-timer-throttling",
            "--disable-ipc-flooding-protection",
            "--disable-hang-monitor",
            "--disable-prompt-on-repost",
            "--disable-client-side-phishing-detection",
            "--disable-webrtc-hw-encoding",
            "--disable-webrtc-hw-decoding",
            "--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
            "--enable-features=NetworkService,NetworkServiceInProcess",
            "--disable-component-update",
            "--disable-domain-reliability",
            "--disable-features=AutofillServerCommunication",
            "--password-store=basic",
            "--use-mock-keychain",
        ]

        self._browser = self._require_playwright().chromium.launch(
            headless=False,
            args=chrome_args,
            ignore_default_args=["--enable-automation", "--no-sandbox"],
        )

        # Create context with HAR recording and realistic settings
        self._context = self._browser.new_context(
            record_har_path=str(self.har_path),
            record_har_content="embed",
            no_viewport=True,
            locale="en-US",
            timezone_id="America/New_York",
            user_agent=self._user_agent,
            screen={"width": 1920, "height": 1080},
            color_scheme="light",
            reduced_motion="no-preference",
            forced_colors="none",
            extra_http_headers={
                "Accept-Language": "en-US,en;q=0.9",
                "sec-ch-ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
                "sec-ch-ua-mobile": "?0",
                "sec-ch-ua-platform": '"macOS"',
            },
        )

        # Apply playwright-stealth evasions
        stealth = Stealth()
        stealth.apply_stealth_sync(self._context)

        # Add custom stealth init script
        self._context.add_init_script(STEALTH_JS)

        # Open initial page
        page = self._context.new_page()

        if start_url:
            page.goto(start_url, wait_until="domcontentloaded")
        else:
            # For HAR recording & context
            page.goto("about:blank")

        # Wait for browser to close
        try:
            while self._context.pages:
                self._context.pages[0].wait_for_timeout(100)
        except Exception:
            pass

        return self.close()

    def start(self, start_url: str | None = None) -> Path:
        """Start the browser with HAR recording enabled. Returns HAR path when done."""
        self._start_time = get_timestamp()

        # Set up signal handlers for graceful shutdown
        signal.signal(signal.SIGINT, self._handle_signal)
        signal.signal(signal.SIGTERM, self._handle_signal)

        console.print(" [dim]capture starting...[/dim]")
        console.print(f" [dim]━[/dim] [white]{self.run_id}[/white]")
        console.print(f" [dim]goal[/dim]  [white]{self.prompt}[/white]")
        console.print()
        console.print(" [dim]navigate and interact to record traffic[/dim]")
        console.print(" [dim]close browser or ctrl+c to finalize[/dim]")
        console.print()

        if start_url:
            start_url = self._normalize_url(start_url)

        self._playwright = sync_playwright().start()

        # Try real Chrome first (better for avoiding detection)
        # Fall back to stealth Chromium if Chrome not available
        try:
            if self.use_real_chrome:
                return self._start_with_real_chrome(start_url)
            else:
                return self._start_with_stealth_chromium(start_url)
        except Exception:
            # Startup failed (e.g. an invalid URL) before close() could stop
            # Playwright. Tear it down here so its sync event loop doesn't leak
            # and break the caller's next asyncio.run() (the REPL prompt).
            self._abort_playwright()
            raise

    def close(self) -> Path:
        """Close the browser and save HAR file. Returns HAR path."""
        end_time = get_timestamp()

        console.print(" [dim]browser closed[/dim]")

        if self._context:
            with Status(
                " [dim]handling har... can take a bit[/dim]",
                console=console,
                spinner="dots",
            ) as status:
                try:
                    status.update(" [dim]flushing network traffic...[/dim]")
                    import time

                    time.sleep(1)

                    status.update(" [dim]saving har file...[/dim]")
                    self._context.close()

                    if self.har_path.exists():
                        har_size = self.har_path.stat().st_size
                        status.update(f" [dim]har saved: {har_size:,} bytes[/dim]")
                    else:
                        console.print(" [yellow]warning: har file was not created[/yellow]")

                except Exception as e:
                    console.print(f" [yellow]warning: error saving har: {e}[/yellow]")
                    if self.har_path.exists():
                        console.print(" [dim]har file exists despite error[/dim]")
                self._context = None

        # Only close browser if not using persistent context
        if self._browser and not self._using_persistent:
            try:
                self._browser.close()
            except Exception:
                pass
            self._browser = None

        if self._playwright:
            try:
                self._playwright.stop()
            except Exception:
                pass
            self._playwright = None

        # Save metadata
        self._save_metadata(end_time)

        console.print(" [dim]capture saved[/dim]")
        console.print(" [dim]metadata synced[/dim]")

        return self.har_path