AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
"""The Playwright fallback must not leak browsers or pages (#26).
A scan that reported 40+ GB of memory on a laptop had the fallback switched on.
Chromium is the likely holder: every "Maybe" result opens a page, and a scan has
hundreds. These tests use a fake Playwright, so they need no browser.
"""
import asyncio
import sys
import types
import pytest
from aliens_eye.core.browser import BrowserFallback
class FakePage:
def __init__(self, world, fail=None, hang=False):
self.world, self.fail, self.hang = world, fail, hang
self.url = "https://example.test/final"
self.closed = False
async def goto(self, url, **kwargs):
self.world.open_pages.add(self)
self.world.peak_open = max(self.world.peak_open, len(self.world.open_pages))
if self.hang:
await asyncio.sleep(3600)
if self.fail:
raise self.fail
await asyncio.sleep(0.01)
async def content(self):
return "<html>ok</html>"
async def close(self):
self.closed = True
self.world.open_pages.discard(self)
class FakeContext:
def __init__(self, world):
self.world, self.closed = world, False
async def new_page(self):
self.world.pages_created += 1
mode = self.world.next_page_mode()
page = FakePage(self.world, **mode)
self.world.all_pages.append(page)
return page
async def close(self):
self.closed = True
# Closing a context closes its pages.
for page in list(self.world.open_pages):
page.closed = True
self.world.open_pages.discard(page)
class FakeBrowser:
def __init__(self, world):
self.world, self.closed = world, False
async def new_context(self, **kwargs):
context = FakeContext(self.world)
self.world.contexts.append(context)
return context
async def close(self):
self.closed = True
class FakeChromium:
def __init__(self, world):
self.world = world
async def launch(self, **kwargs):
await asyncio.sleep(0.05) # starting a real browser takes a while
browser = FakeBrowser(self.world)
self.world.browsers.append(browser)
return browser
class FakePlaywright:
def __init__(self, world):
self.chromium = FakeChromium(world)
async def stop(self):
pass
class World:
def __init__(self):
self.browsers, self.contexts, self.all_pages = [], [], []
self.open_pages, self.peak_open, self.pages_created = set(), 0, 0
self.modes = []
def next_page_mode(self):
return self.modes.pop(0) if self.modes else {}
@pytest.fixture
def world(monkeypatch):
world = World()
class Starter:
async def start(self):
await asyncio.sleep(0.05)
return FakePlaywright(world)
module = types.ModuleType("playwright.async_api")
module.async_playwright = lambda: Starter()
package = types.ModuleType("playwright")
package.async_api = module
monkeypatch.setitem(sys.modules, "playwright", package)
monkeypatch.setitem(sys.modules, "playwright.async_api", module)
return world
async def test_concurrent_first_fetches_start_one_browser(world):
"""Regression: 50 workers hitting a Maybe together each launched a Chromium."""
fallback = BrowserFallback(timeout=5, user_agent="test")
await asyncio.gather(*(fallback.fetch(f"https://example.test/{i}") for i in range(30)))
assert len(world.browsers) == 1
assert len(world.contexts) == 1
await fallback.close()
assert world.browsers[0].closed
async def test_a_page_that_fails_to_load_is_still_closed(world):
world.modes = [{"fail": RuntimeError("net::ERR_TIMED_OUT")}]
fallback = BrowserFallback(timeout=5, user_agent="test")
with pytest.raises(RuntimeError):
await fallback.fetch("https://example.test/a")
assert world.all_pages[0].closed
assert world.open_pages == set()
async def test_a_cancelled_fetch_closes_its_page(world):
"""The scan gives each site a deadline and cancels it; the page must not outlive that."""
world.modes = [{"hang": True}]
fallback = BrowserFallback(timeout=5, user_agent="test")
task = asyncio.create_task(fallback.fetch("https://example.test/slow"))
await asyncio.sleep(0.3)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert world.all_pages[0].closed
assert world.open_pages == set()
async def test_one_page_is_open_at_a_time(world):
fallback = BrowserFallback(timeout=5, user_agent="test")
await asyncio.gather(*(fallback.fetch(f"https://example.test/{i}") for i in range(20)))
assert world.peak_open == 1
async def test_the_context_is_recycled_so_browser_memory_stays_bounded(world, monkeypatch):
monkeypatch.setattr("aliens_eye.core.browser.RECYCLE_AFTER_PAGES", 5)
fallback = BrowserFallback(timeout=5, user_agent="test")
for i in range(12):
await fallback.fetch(f"https://example.test/{i}")
assert len(world.browsers) == 1 # the browser is reused
assert len(world.contexts) == 3 # 12 pages at 5 per context
assert [c.closed for c in world.contexts] == [True, True, False]
await fallback.close()
assert all(c.closed for c in world.contexts)
async def test_a_failed_close_does_not_hide_the_result(world, monkeypatch):
fallback = BrowserFallback(timeout=5, user_agent="test")
async def broken_close(self):
raise RuntimeError("target closed")
monkeypatch.setattr(FakePage, "close", broken_close)
content, final_url = await fallback.fetch("https://example.test/a")
assert content == "<html>ok</html>" and final_url == "https://example.test/final"
async def test_close_is_safe_when_never_started(world):
await BrowserFallback(timeout=5, user_agent="test").close()
async def test_fetch_after_close_starts_fresh(world):
fallback = BrowserFallback(timeout=5, user_agent="test")
await fallback.fetch("https://example.test/a")
await fallback.close()
await fallback.fetch("https://example.test/b")
assert len(world.browsers) == 2
async def test_a_browser_that_fails_halfway_through_starting_is_shut_down(world, monkeypatch):
"""The browser launched, the context did not: the browser must not be left running."""
async def broken_context(self, **kwargs):
raise RuntimeError("could not create a context")
monkeypatch.setattr(FakeBrowser, "new_context", broken_context)
fallback = BrowserFallback(timeout=5, user_agent="test")
with pytest.raises(RuntimeError):
await fallback.fetch("https://example.test/a")
assert len(world.browsers) == 1 and world.browsers[0].closed
async def test_a_start_cancelled_by_the_scan_stops_its_driver(world, monkeypatch):
"""The scan's per-site deadline cancels a check wherever it happens to be.
Playwright spawns its driver before start() returns it, so a start
interrupted in the middle leaves that driver running with nothing holding
it -- file descriptors the scan never gets back (#26).
"""
starters = []
class SlowStarter:
def __init__(self):
self.stopped = False
starters.append(self)
async def start(self):
await asyncio.sleep(3600) # cancelled while waiting in here
async def __aexit__(self, *args):
self.stopped = True
monkeypatch.setattr(sys.modules["playwright.async_api"], "async_playwright", SlowStarter)
fallback = BrowserFallback(timeout=5, user_agent="test")
task = asyncio.create_task(fallback.fetch("https://example.test/slow"))
await asyncio.sleep(0.1)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert starters and starters[0].stopped
async def test_a_driver_that_cannot_start_is_stopped_and_can_be_retried(world, monkeypatch):
"""Errno 24 at spawn -- the failure in the issue's follow-up log.
The half-started Playwright must be stopped, and because no context was
ever set the next Maybe must be able to try again from scratch.
"""
starters, attempts = [], []
class FlakyStarter:
def __init__(self):
self.stopped = False
starters.append(self)
async def start(self):
attempts.append(1)
if len(attempts) == 1:
raise OSError(24, "Too many open files")
return FakePlaywright(world)
async def __aexit__(self, *args):
self.stopped = True
monkeypatch.setattr(sys.modules["playwright.async_api"], "async_playwright", FlakyStarter)
fallback = BrowserFallback(timeout=5, user_agent="test")
with pytest.raises(OSError):
await fallback.fetch("https://example.test/a")
assert starters[0].stopped
assert fallback._context is None
content, final_url = await fallback.fetch("https://example.test/b")
assert content == "<html>ok</html>"
assert len(starters) == 2 and len(world.browsers) == 1
await fallback.close()