Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

fix: the scan held hundreds of sockets and hid a failing fallback (#26); release 2.7.4

arxhr007021eaa1c8a77
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0746e87..082db13 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,28 @@
# Changelog
+## 2.7.4 (2026-10-20)
+
+### Fixed
+- **A scan could run the process out of file descriptors** (#26, the follow-up log:
+ `Task exception was never retrieved ... OSError: [Errno 24] Too many open files`).
+ Three things added up:
+ - aiohttp holds each finished request's socket in a pool for 75 seconds (its default),
+ one per site, while the connector's `limit` counts only the connections in use.
+ Scanning 854 sites therefore left several hundred idle sockets open at once -- past
+ macOS's default of 256 file descriptors, after which every socket, and every attempt
+ to spawn the Playwright driver, failed with Errno 24. Idle sockets are now reaped
+ after 10 seconds, and the CLI raises the soft limit to 4096 at startup where the OS
+ allows it.
+ - Errno 24 in a fetch read only "Too many open files", like the site had failed. It now
+ says the process hit its file-descriptor limit and names `ulimit -n`.
+ - A browser fallback that could not start was logged at debug level, so the scan looked
+ healthy while none of its "Maybe" results were re-checked in a browser. The first
+ failure is now warned about once per scan.
+- **A start of Playwright interrupted by the scan's per-site deadline left its driver
+ process and pipes running**, held by nothing: leaked file descriptors, and the
+ fire-and-forgotten `Connection.run()` task that printed "Task exception was never
+ retrieved". A start that fails or is cancelled now stops the driver it already spawned.
+
## 2.7.3 (2026-10-20)
### Fixed
diff --git a/pyproject.toml b/pyproject.toml
index ea658d2..9c95701 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "aliens-eye"
-version = "2.7.3"
+version = "2.7.4"
description = "AI-powered OSINT username scanner across 800+ social media and web platforms"
readme = "README.md"
requires-python = ">=3.10"
diff --git a/src/aliens_eye/__init__.py b/src/aliens_eye/__init__.py
index 4bb3169..84793bd 100644
--- a/src/aliens_eye/__init__.py
+++ b/src/aliens_eye/__init__.py
@@ -1,3 +1,3 @@
"""Aliens Eye - AI-powered OSINT username scanner."""
-__version__ = "2.7.3"
+__version__ = "2.7.4"
diff --git a/src/aliens_eye/cli.py b/src/aliens_eye/cli.py
index 1b18062..d366b91 100644
--- a/src/aliens_eye/cli.py
+++ b/src/aliens_eye/cli.py
@@ -28,6 +28,7 @@ from aliens_eye.core.scanner import (
load_sites_data,
)
from aliens_eye.utils.console import get_console, set_plain
+from aliens_eye.utils.fdlimit import raise_fd_limit
from aliens_eye.utils.logger import setup_logger
TOR_PROXY = "socks5://127.0.0.1:9050"
@@ -1418,6 +1419,7 @@ def main() -> None:
if sys.platform == "win32":
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
sys.stderr.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
+ raise_fd_limit() # macOS hands out 256; one scan can pass that (#26)
argv = sys.argv[1:]
console = get_console()
try:
diff --git a/src/aliens_eye/core/browser.py b/src/aliens_eye/core/browser.py
index 7d75ebd..a0deaef 100644
--- a/src/aliens_eye/core/browser.py
+++ b/src/aliens_eye/core/browser.py
@@ -6,6 +6,22 @@ RECYCLE_AFTER_PAGES = 40
CLOSE_TIMEOUT = 5.0
+async def _stop_starter(starter) -> None:
+ """Stop a Playwright whose ``start()`` never returned it.
+
+ ``async_playwright().start()`` spawns the driver before it returns, so a
+ start that fails or is cancelled part-way through leaves that driver
+ running. The context manager's own exit is the only handle on it. What it
+ raises does not matter here: the exception being propagated is already on
+ its way out, and a Playwright that never got far enough to stop simply
+ had nothing running.
+ """
+ try:
+ await starter.__aexit__()
+ except Exception: # noqa: BLE001 - best effort cleanup on the way out
+ pass
+
+
class BrowserFallback:
"""Playwright-based fallback for hard pages.
@@ -27,12 +43,19 @@ class BrowserFallback:
except ImportError as exc:
raise RuntimeError("Playwright is not installed") from exc
- self._playwright = await async_playwright().start()
+ starter = async_playwright()
try:
+ # Every await of the start is inside the guard, including the
+ # first: the scan's per-site deadline cancels a check wherever it
+ # happens to be, and a Playwright whose start() never returned
+ # keeps its driver process and pipes open with nothing holding
+ # them. Each orphan is file descriptors the scan never gets back.
+ self._playwright = await starter.start()
self._browser = await self._playwright.chromium.launch(headless=True)
self._context = await self._browser.new_context(user_agent=self.user_agent)
except BaseException:
await self.close() # never leave a half-started browser running
+ await _stop_starter(starter)
raise
self._pages_in_context = 0
diff --git a/src/aliens_eye/core/http.py b/src/aliens_eye/core/http.py
index a5addd5..59760ac 100644
--- a/src/aliens_eye/core/http.py
+++ b/src/aliens_eye/core/http.py
@@ -1,4 +1,5 @@
import asyncio
+import errno
import random
import time
from dataclasses import dataclass
@@ -9,6 +10,14 @@ import aiohttp
from .config import ScannerConfig
from .rate_limit import DomainRateLimiter
+# How long a finished request's socket may sit idle in the pool. aiohttp's
+# default is 75s, and the pool holds one idle connection per site for that
+# long: a scan of 854 sites then had hundreds open at once, past macOS's
+# default of 256 file descriptors, and every socket and subprocess spawn after
+# that failed with Errno 24 (#26). The ``limit`` of the connector does not
+# bound these -- it counts only the connections currently in use.
+KEEPALIVE_SECONDS = 10.0
+
@dataclass
class FetchResult:
@@ -66,6 +75,21 @@ def _parse_retry_after(headers: dict[str, str]) -> float | None:
return seconds
+def describe_error(exc: BaseException) -> str:
+ """A fetch failure as text, naming the fix when there is an obvious one.
+
+ Errno 24 means the process-wide file-descriptor limit was hit. It reaches
+ us from aiohttp's socket and from the browser fallback's driver spawn alike,
+ and as "Too many open files" alone it reads like the site failed (#26).
+ """
+ if isinstance(exc, OSError) and exc.errno in (errno.EMFILE, errno.ENFILE):
+ return (
+ f"{exc.strerror or exc} (the process hit its file-descriptor limit; "
+ "macOS starts programs at 256 -- raise it with e.g. `ulimit -n 4096`)"
+ )
+ return str(exc)
+
+
async def fetch_url(
session: aiohttp.ClientSession,
url: str,
@@ -133,7 +157,11 @@ async def fetch_url(
last_status = exc.status or last_status
retry_after = _parse_retry_after(last_headers)
except aiohttp.ClientError as exc:
- error_message = str(exc)
+ # ClientOSError is one of these: a refused or exhausted socket
+ # arrives here, errno included.
+ error_message = describe_error(exc)
+ except OSError as exc:
+ error_message = describe_error(exc)
except Exception as exc:
error_message = str(exc)
diff --git a/src/aliens_eye/core/scanner.py b/src/aliens_eye/core/scanner.py
index 5a7920b..cf9fd77 100644
--- a/src/aliens_eye/core/scanner.py
+++ b/src/aliens_eye/core/scanner.py
@@ -17,7 +17,7 @@ from .config import DEFAULT_HEADERS, MAX_HEADER_SIZE, ScannerConfig
from .control import PageShape, control_username, http_overrule, page_shape, same_page
from .detector import Detector
from .fingerprints import FingerprintStore, build_fingerprint
-from .http import fetch_url
+from .http import KEEPALIVE_SECONDS, describe_error, fetch_url
from .rate_limit import DomainRateLimiter
from .variations import generate_username_variations, scan_cost
@@ -163,14 +163,20 @@ def build_connector(
``resolver`` is for fetches of URLs taken from scanned pages; see
``core.netguard.PublicOnlyResolver``. It has no effect through a SOCKS
proxy, where the proxy does the resolving.
+
+ ``keepalive_timeout`` is kept short on purpose; see ``KEEPALIVE_SECONDS``.
"""
if config.proxy and config.proxy.lower().startswith(("socks4://", "socks5://")):
from aiohttp_socks import ProxyConnector
- return ProxyConnector.from_url(config.proxy, limit=limit)
+ return ProxyConnector.from_url(
+ config.proxy, limit=limit, keepalive_timeout=KEEPALIVE_SECONDS
+ )
if resolver is not None:
- return aiohttp.TCPConnector(limit=limit, resolver=resolver)
- return aiohttp.TCPConnector(limit=limit)
+ return aiohttp.TCPConnector(
+ limit=limit, resolver=resolver, keepalive_timeout=KEEPALIVE_SECONDS
+ )
+ return aiohttp.TCPConnector(limit=limit, keepalive_timeout=KEEPALIVE_SECONDS)
def build_session(
@@ -251,6 +257,11 @@ class UsernameScanner:
# first time that site reports Found. None: it could not be fetched.
self._controls: dict[str, PageShape | None] = {}
+ # One warning per scan for a browser fallback that will not start.
+ # Hundreds of Maybes each fail the same way, and at debug level the
+ # whole scan looked healthy while none of them were re-checked (#26).
+ self._fallback_warned = False
+
self.results_dir = config.output_dir
self.results_dir.mkdir(parents=True, exist_ok=True)
@@ -580,7 +591,19 @@ class UsernameScanner:
try:
content, final_url = await self.browser_fallback.fetch(url)
except Exception as exc:
- self.logger.debug("Playwright fallback failed for %s: %s", site_name, exc)
+ if not self._fallback_warned:
+ self._fallback_warned = True
+ self.logger.warning(
+ "Browser fallback failed for %s (%s); Maybe results are "
+ "not re-checked in a browser (further failures are logged "
+ "at debug)",
+ site_name,
+ describe_error(exc),
+ )
+ else:
+ self.logger.debug(
+ "Playwright fallback failed for %s: %s", site_name, exc
+ )
return status_text, confidence, ai_analysis
bundle = self.extractor.extract(
diff --git a/src/aliens_eye/utils/fdlimit.py b/src/aliens_eye/utils/fdlimit.py
new file mode 100644
index 0000000..63e459f
--- /dev/null
+++ b/src/aliens_eye/utils/fdlimit.py
@@ -0,0 +1,34 @@
+"""The per-process file-descriptor limit.
+
+macOS starts a program with 256, and a single scan can pass that: one pooled
+socket per site plus the browser fallback's driver pipes, after which every
+socket and subprocess spawn fails with Errno 24. Linux usually starts far
+higher, which is why the failure looked machine-specific (#26).
+"""
+
+import sys
+
+
+def raise_fd_limit(target: int = 4096) -> int | None:
+ """Raise the soft file-descriptor limit to ``target``, as far as allowed.
+
+ Returns the limit in force, or None where the platform cannot ask (Windows)
+ or the hard limit is lower. Never raises: a program that cannot raise its
+ limit should still run, and the error it eventually gets names the fix.
+ """
+ if sys.platform == "win32":
+ return None
+ try:
+ import resource
+ except ImportError: # pragma: no cover - POSIX without the resource module
+ return None
+ try:
+ soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+ if hard != resource.RLIM_INFINITY:
+ target = min(target, hard)
+ if target > soft:
+ resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard))
+ soft = target
+ return soft
+ except (OSError, ValueError): # a platform that refuses; run as we were given
+ return None
diff --git a/tests/test_browser.py b/tests/test_browser.py
index 00bd488..3536e06 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -199,3 +199,69 @@ async def test_a_browser_that_fails_halfway_through_starting_is_shut_down(world,
with pytest.raises(RuntimeError):
await fallback.fetch("https://example.test/a")
assert len(world.browsers) == 1 and world.browsers[0].closed
+
+
+async def test_a_start_cancelled_by_the_scan_stops_its_driver(world, monkeypatch):
+ """The scan's per-site deadline cancels a check wherever it happens to be.
+
+ Playwright spawns its driver before start() returns it, so a start
+ interrupted in the middle leaves that driver running with nothing holding
+ it -- file descriptors the scan never gets back (#26).
+ """
+ starters = []
+
+ class SlowStarter:
+ def __init__(self):
+ self.stopped = False
+ starters.append(self)
+
+ async def start(self):
+ await asyncio.sleep(3600) # cancelled while waiting in here
+
+ async def __aexit__(self, *args):
+ self.stopped = True
+
+ monkeypatch.setattr(sys.modules["playwright.async_api"], "async_playwright", SlowStarter)
+ fallback = BrowserFallback(timeout=5, user_agent="test")
+ task = asyncio.create_task(fallback.fetch("https://example.test/slow"))
+ await asyncio.sleep(0.1)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert starters and starters[0].stopped
+
+
+async def test_a_driver_that_cannot_start_is_stopped_and_can_be_retried(world, monkeypatch):
+ """Errno 24 at spawn -- the failure in the issue's follow-up log.
+
+ The half-started Playwright must be stopped, and because no context was
+ ever set the next Maybe must be able to try again from scratch.
+ """
+ starters, attempts = [], []
+
+ class FlakyStarter:
+ def __init__(self):
+ self.stopped = False
+ starters.append(self)
+
+ async def start(self):
+ attempts.append(1)
+ if len(attempts) == 1:
+ raise OSError(24, "Too many open files")
+ return FakePlaywright(world)
+
+ async def __aexit__(self, *args):
+ self.stopped = True
+
+ monkeypatch.setattr(sys.modules["playwright.async_api"], "async_playwright", FlakyStarter)
+ fallback = BrowserFallback(timeout=5, user_agent="test")
+
+ with pytest.raises(OSError):
+ await fallback.fetch("https://example.test/a")
+ assert starters[0].stopped
+ assert fallback._context is None
+
+ content, final_url = await fallback.fetch("https://example.test/b")
+ assert content == "<html>ok</html>"
+ assert len(starters) == 2 and len(world.browsers) == 1
+ await fallback.close()
diff --git a/tests/test_fdlimit.py b/tests/test_fdlimit.py
new file mode 100644
index 0000000..c571dca
--- /dev/null
+++ b/tests/test_fdlimit.py
@@ -0,0 +1,17 @@
+"""macOS starts a program with 256 file descriptors; one scan can pass that.
+
+The follow-up report on #26 ended in Errno 24 from every socket and subprocess
+spawn afterwards. The limit is raised at startup where the OS allows it, and
+asking must never itself be a way to fail.
+"""
+
+from aliens_eye.utils.fdlimit import raise_fd_limit
+
+
+def test_the_limit_is_raised_or_reported():
+ limit = raise_fd_limit()
+ assert limit is None or limit >= 256
+
+
+def test_asking_twice_leaves_the_same_limit():
+ assert raise_fd_limit() == raise_fd_limit()
diff --git a/tests/test_http.py b/tests/test_http.py
index 96c3f2c..c04f10a 100644
--- a/tests/test_http.py
+++ b/tests/test_http.py
@@ -1,6 +1,7 @@
"""Body reading: the whole page, not whatever happened to arrive first."""
import asyncio
+import errno
import aiohttp
import pytest
@@ -8,7 +9,7 @@ from aiohttp import web
from aiohttp.test_utils import TestServer
from aliens_eye.core.config import ScannerConfig
-from aliens_eye.core.http import fetch_url
+from aliens_eye.core.http import describe_error, fetch_url
from aliens_eye.core.rate_limit import DomainRateLimiter
HEAD = "<html><head><title>t</title>" + "<!-- padding -->" * 3000 # ~48 KB of head
@@ -125,3 +126,29 @@ async def test_a_short_retry_after_is_honoured_once_not_twice(busy_server, logge
await fetch_busy(busy_server, logger, "0.4", retries=1, backoff_base=0.0)
first, second = busy_server.hits
assert 0.35 <= second - first < 0.7
+
+
+# --- Errno 24: the process ran out of file descriptors, the site did not fail ----------
+
+
+class ExhaustedSession:
+ """A session whose very first socket fails the way a full process does."""
+
+ def get(self, *args, **kwargs):
+ raise OSError(errno.EMFILE, "Too many open files")
+
+
+async def test_running_out_of_file_descriptors_names_the_fix(logger):
+ """A scan of 854 sites passed macOS's 256 descriptors; the error that
+ followed said only "Too many open files", like the site had failed (#26)."""
+ config = ScannerConfig(retries=0, rate_limit_delay=0.0)
+ result = await fetch_url(
+ ExhaustedSession(), "https://example.test/a", config, DomainRateLimiter(), logger
+ )
+ assert result.error and "file-descriptor" in result.error
+ assert "ulimit" in result.error
+
+
+def test_a_ordinary_error_is_reported_verbatim():
+ assert describe_error(aiohttp.ClientError("boom")) == "boom"
+ assert describe_error(ConnectionResetError("peer reset")) == "peer reset"
diff --git a/tests/test_scanner.py b/tests/test_scanner.py
index 4a2a2a9..e4ff8eb 100644
--- a/tests/test_scanner.py
+++ b/tests/test_scanner.py
@@ -146,6 +146,66 @@ async def test_build_connector_http_proxy_uses_tcp():
await connector.close()
+async def test_idle_sockets_are_reaped_within_seconds():
+ """aiohttp pools one idle connection per site, uncounted by ``limit``.
+
+ Held for its default 75s, a scan of 854 sites had hundreds of these open
+ at once -- past macOS's default of 256 file descriptors, after which every
+ socket and the browser fallback's driver spawn failed with Errno 24 (#26).
+ """
+ connector = build_connector(ScannerConfig(), 5)
+ try:
+ assert connector._keepalive_timeout <= 15
+ finally:
+ await connector.close()
+
+
+# --- a browser fallback that will not start ---------------------------------------------
+
+
+class BrokenFallback:
+ def __init__(self):
+ self.calls = 0
+
+ async def fetch(self, url):
+ self.calls += 1
+ raise OSError(24, "Too many open files")
+
+
+class RecordingLogger:
+ """Keeps every record: the test counts them."""
+
+ def __init__(self):
+ self.records = []
+
+ def warning(self, msg, *args):
+ self.records.append(("WARNING", msg % args if args else msg))
+
+ def debug(self, msg, *args):
+ self.records.append(("DEBUG", msg % args if args else msg))
+
+
+async def test_a_fallback_that_will_not_start_is_warned_about_once(site_server, tmp_path, logger):
+ """Every Maybe failed the same way, at debug level, and the scan looked
+ healthy while none of them were re-checked in a browser (#26)."""
+ scanner = make_scanner(server_sites(site_server), tmp_path, logger)
+ scanner.browser_fallback = BrokenFallback()
+ recorder = RecordingLogger()
+ scanner.logger = recorder
+
+ for _ in range(3):
+ outcome = await scanner._playwright_fallback(
+ "alpha", "someone", "http://example.test/a", None, {"method": "x"}, "Maybe", 40
+ )
+ assert outcome == ("Maybe", 40, {"method": "x"}) # unchanged, as promised
+
+ warnings = [msg for level, msg in recorder.records if level == "WARNING"]
+ assert len(warnings) == 1
+ assert "file-descriptor" in warnings[0]
+ assert sum(1 for level, _ in recorder.records if level == "DEBUG") == 2
+ assert scanner.browser_fallback.calls == 3 # it is still retried
+
+
# --- bot-wall cap ---------------------------------------------------------
from aliens_eye.core.scanner import _looks_like_bot_wall # noqa: E402