Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

10.8 KB
Raw Blame
import socket

import aiohttp
import pytest
from aiohttp import web
from aiohttp.test_utils import TestServer

from aliens_eye.core.analyzer import FeatureExtractor
from aliens_eye.core.config import ScannerConfig
from aliens_eye.core.detector import Detector
from aliens_eye.core.fingerprints import FingerprintStore
from aliens_eye.core.scanner import UsernameScanner, build_connector, format_site_url


def free_port() -> int:
    with socket.socket() as sock:
        sock.bind(("127.0.0.1", 0))
        return sock.getsockname()[1]


@pytest.fixture
async def site_server(found_html, not_found_html):
    app = web.Application()

    async def alpha(request):
        # A site that knows its users: anyone else gets its "no such user" page.
        if request.match_info["username"] not in {"torvalds", "ghost"}:
            return web.Response(text=not_found_html, status=404, content_type="text/html")
        return web.Response(text=found_html, content_type="text/html")

    async def beta(request):
        return web.Response(text=not_found_html, status=404, content_type="text/html")

    app.router.add_get("/alpha/{username}", alpha)
    app.router.add_get("/beta/users/{username}", beta)
    server = TestServer(app)
    await server.start_server()
    yield server
    await server.close()


def make_scanner(sites, tmp_path, logger):
    config = ScannerConfig(
        retries=0,
        rate_limit_delay=0.0,
        output_dir=tmp_path / "results",
        fingerprints_path=tmp_path / "fp.json",
        plain_output=True,
    )
    return UsernameScanner(
        sites_data=sites,
        config=config,
        extractor=FeatureExtractor(),
        detector=Detector(),
        fingerprints=FingerprintStore(config.fingerprints_path),
        logger=logger,
    )


def server_sites(server):
    base = f"http://{server.host}:{server.port}"
    return {
        "alpha": base + "/alpha/{}",
        "beta": base + "/beta/users/{}",
    }


async def test_scan_all_sites_end_to_end(site_server, tmp_path, logger):
    scanner = make_scanner(server_sites(site_server), tmp_path, logger)
    results = await scanner.scan_all_sites("torvalds")

    assert len(results) == 2
    by_site = {r["site"]: r for r in results}
    assert by_site["alpha"]["status"] == "Found"
    assert by_site["alpha"]["code"] == 200
    assert by_site["beta"]["status"] == "Not Found"
    assert by_site["beta"]["code"] == 404
    for r in results:
        assert set(r) >= {"site", "url", "status", "code", "confidence", "ai_analysis"}


async def test_scan_handles_connection_error(site_server, tmp_path, logger):
    sites = server_sites(site_server)
    sites["dead"] = f"http://127.0.0.1:{free_port()}/{{}}"
    scanner = make_scanner(sites, tmp_path, logger)
    results = await scanner.scan_all_sites("ghost")

    by_site = {r["site"]: r for r in results}
    assert by_site["dead"]["status"] in {"Error", "Timeout"}
    assert by_site["alpha"]["status"] == "Found"


async def test_scan_with_variations_basic(site_server, tmp_path, logger):
    scanner = make_scanner(server_sites(site_server), tmp_path, logger)
    all_results = await scanner.scan_with_variations("torvalds", "basic")

    assert list(all_results) == ["torvalds"]
    assert len(all_results["torvalds"]) == 2


def test_format_url_fallbacks():
    assert (
        UsernameScanner._format_url("x", "https://x.com/{}", "bob") == "https://x.com/bob"
    )
    assert (
        UsernameScanner._format_url("x", "https://x.com/{user}", "bob")
        == "https://x.com/bob"
    )


def test_format_site_url_percent_encodes_non_ascii():
    url = format_site_url("weibo", "https://weibo.com/{}", "坦途")
    assert url == "https://weibo.com/%E5%9D%A6%E9%80%94"
    assert "坦途" not in url


def test_format_site_url_leaves_ascii_handles_unchanged():
    assert format_site_url("x", "https://x.com/{}", "bob_dev-99") == "https://x.com/bob_dev-99"


def test_format_site_url_encodes_special_characters():
    url = format_site_url("x", "https://x.com/{}", "a b@c")
    assert url == "https://x.com/a%20b%40c"


async def test_build_connector_tcp():
    config = ScannerConfig()
    connector = build_connector(config, 5)
    assert isinstance(connector, aiohttp.TCPConnector)
    await connector.close()


async def test_build_connector_socks():
    from aiohttp_socks import ProxyConnector

    config = ScannerConfig(proxy="socks5://127.0.0.1:9050")
    connector = build_connector(config, 5)
    assert isinstance(connector, ProxyConnector)
    await connector.close()


async def test_build_connector_http_proxy_uses_tcp():
    config = ScannerConfig(proxy="http://127.0.0.1:8080")
    connector = build_connector(config, 5)
    assert isinstance(connector, aiohttp.TCPConnector)
    await connector.close()


async def test_idle_sockets_are_reaped_within_seconds():
    """aiohttp pools one idle connection per site, uncounted by ``limit``.

    Held for its default 75s, a scan of 854 sites had hundreds of these open
    at once -- past macOS's default of 256 file descriptors, after which every
    socket and the browser fallback's driver spawn failed with Errno 24 (#26).
    """
    connector = build_connector(ScannerConfig(), 5)
    try:
        assert connector._keepalive_timeout <= 15
    finally:
        await connector.close()


# --- a browser fallback that will not start ---------------------------------------------


class BrokenFallback:
    def __init__(self):
        self.calls = 0

    async def fetch(self, url):
        self.calls += 1
        raise OSError(24, "Too many open files")


class RecordingLogger:
    """Keeps every record: the test counts them."""

    def __init__(self):
        self.records = []

    def warning(self, msg, *args):
        self.records.append(("WARNING", msg % args if args else msg))

    def debug(self, msg, *args):
        self.records.append(("DEBUG", msg % args if args else msg))


async def test_a_fallback_that_will_not_start_is_warned_about_once(site_server, tmp_path, logger):
    """Every Maybe failed the same way, at debug level, and the scan looked
    healthy while none of them were re-checked in a browser (#26)."""
    scanner = make_scanner(server_sites(site_server), tmp_path, logger)
    scanner.browser_fallback = BrokenFallback()
    recorder = RecordingLogger()
    scanner.logger = recorder

    for _ in range(3):
        outcome = await scanner._playwright_fallback(
            "alpha", "someone", "http://example.test/a", None, {"method": "x"}, "Maybe", 40
        )
        assert outcome == ("Maybe", 40, {"method": "x"})   # unchanged, as promised

    warnings = [msg for level, msg in recorder.records if level == "WARNING"]
    assert len(warnings) == 1
    assert "file-descriptor" in warnings[0]
    assert sum(1 for level, _ in recorder.records if level == "DEBUG") == 2
    assert scanner.browser_fallback.calls == 3             # it is still retried


# --- bot-wall cap ---------------------------------------------------------

from aliens_eye.core.scanner import _looks_like_bot_wall  # noqa: E402


@pytest.mark.parametrize("title", [
    "Just a moment...",
    "Checking your browser",
    "Attention Required! | Cloudflare",
    "Access Denied",
    "500 Internal Server Error",
    "Verify you are human",
])
def test_challenge_and_error_titles_are_bot_walls(title):
    assert _looks_like_bot_wall({"title": title, "meta_samples": []})


def test_challenge_text_in_meta_tags_counts():
    assert _looks_like_bot_wall({"title": "example.com", "meta_samples": ["Please complete the CAPTCHA"]})


@pytest.mark.parametrize("signals", [
    {"title": "torvalds (Linus Torvalds) - Profile", "meta_samples": ["followers and posts"]},
    {"title": "", "meta_samples": []},
    {"title": None},
    {},
])
def test_ordinary_pages_are_not_bot_walls(signals):
    assert not _looks_like_bot_wall(signals)


async def test_a_challenge_page_is_capped_at_maybe(found_html, tmp_path, logger):
    """A 200 page with profile wording but a challenge title must not be Found.

    Seen live: pcgamer served "Checking your browser" for every username, real
    or made up, and each one was reported as a found account.
    """
    app = web.Application()
    challenge = found_html.replace(
        "<title>torvalds (Linus Torvalds) - Profile</title>",
        "<title>Checking your browser</title>",
    )
    assert challenge != found_html

    async def page(request):
        return web.Response(text=challenge, content_type="text/html")

    app.router.add_get("/u/{username}", page)
    server = TestServer(app)
    await server.start_server()
    try:
        sites = {"walled": f"http://{server.host}:{server.port}/u/{{}}"}
        [result] = await make_scanner(sites, tmp_path, logger).scan_all_sites("torvalds")
    finally:
        await server.close()

    assert result["status"] == "Maybe"
    assert result["confidence"] <= 55
    # The suffix is only added when the uncapped verdict was Found, so this also
    # proves the cap is what changed the result.
    assert result["ai_analysis"]["method"].endswith("-botwall-capped")


async def test_a_cancelled_scan_leaves_no_workers_behind(tmp_path, logger):
    """Regression: cancelling a scan (a caller's timeout) left its workers pending forever."""
    import asyncio

    app = web.Application()

    async def stall(request):
        await asyncio.sleep(30)
        return web.Response(text="late")

    app.router.add_get("/slow/{username}", stall)
    server = TestServer(app)
    await server.start_server()
    try:
        base = f"http://{server.host}:{server.port}"
        scanner = make_scanner({f"s{i}": base + "/slow/{}" for i in range(4)}, tmp_path, logger)
        before = asyncio.all_tasks()
        with pytest.raises(asyncio.TimeoutError):
            await asyncio.wait_for(scanner.scan_all_sites("torvalds"), 0.3)
        await asyncio.sleep(0.05)
        left = [t for t in asyncio.all_tasks() - before if "_worker" in repr(t.get_coro())]
        assert left == []
    finally:
        await server.close()


async def test_one_site_that_never_answers_cannot_hold_up_the_scan(site_server, tmp_path, logger):
    """A real scan sat on its last request for half an hour. Whatever the cause
    inside one site's check, the scan moves on after a bounded wait."""
    import asyncio

    from aliens_eye.core.http import fetch_url

    scanner = make_scanner(server_sites(site_server), tmp_path, logger)
    scanner.config.timeout = 0.2
    scanner.config.backoff_cap = 0.0

    async def fetch(session, url, config, rate_limiter, log):
        if "/beta/" in url:
            await asyncio.sleep(3600)      # ignores every timeout the HTTP layer has
        return await fetch_url(session, url, config, rate_limiter, log)

    scanner.fetch = fetch
    results = await asyncio.wait_for(scanner.scan_all_sites("torvalds"), 10)
    by_site = {r["site"]: r for r in results}
    assert by_site["alpha"]["status"] == "Found"
    assert by_site["beta"]["status"] == "Timeout" and by_site["beta"]["code"] == 408