AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
fix: declare Brotli and advertise only decodable content encodings
arxhr007401f0e2cfdab
diff --git a/pyproject.toml b/pyproject.tomlindex c51c822..b14c054 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -26,6 +26,9 @@ classifiers = []dependencies = ["aiohttp>=3.8",+ # Required, not optional: DEFAULT_HEADERS advertises br, and a server that+ # honours it returns a body aiohttp raises on without a Brotli decoder.+ "Brotli>=1.1","selectolax>=0.3.21","rich>=13.7","aiohttp-socks>=0.8,<1.0",diff --git a/src/aliens_eye/core/config.py b/src/aliens_eye/core/config.pyindex 03e1ea0..368304f 100644--- a/src/aliens_eye/core/config.py+++ b/src/aliens_eye/core/config.py@@ -10,10 +10,32 @@ DEFAULT_USER_AGENT = ("AppleWebKit/537.36 Chrome/115.0.0.0 Safari/537.36")+def _supported_encodings() -> str:+ """Advertise only what this install can actually decode.++ Claiming ``br`` without a Brotli decoder present is not a degraded response,+ it is a hard failure: any server that honours it returns a body aiohttp+ raises on ("can not decode content-encoding: brotli"), so the site is scanned+ as an error rather than checked. Brotli is a declared dependency, but an+ existing environment may predate that, so the header is built from what is+ importable rather than assumed.+ """+ encodings = ["gzip", "deflate"]+ try: # pragma: no cover - exercised by whichever branch the env provides+ import brotli # noqa: F401+ except ImportError:+ try:+ import brotlicffi # noqa: F401+ except ImportError:+ return ", ".join(encodings)+ encodings.append("br")+ return ", ".join(encodings)++DEFAULT_HEADERS = {"User-Agent": DEFAULT_USER_AGENT,"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",- "Accept-Encoding": "gzip, deflate, br",+ "Accept-Encoding": _supported_encodings(),"Accept-Language": "en-US,en;q=0.5","Connection": "keep-alive","DNT": "1",diff --git a/tests/test_config_cli.py b/tests/test_config_cli.pyindex 524c87a..7a484b7 100644--- a/tests/test_config_cli.py+++ b/tests/test_config_cli.py@@ -98,3 +98,45 @@ def test_load_config_explicit_path(tmp_path, logger):def test_load_config_missing_returns_empty(tmp_path, logger):assert load_config(str(tmp_path / "nope.json"), logger) == {}+++def test_accept_encoding_only_advertises_decodable_encodings():+ """Advertising br without a decoder is a hard failure, not a degradation.++ Regression: DEFAULT_HEADERS hardcoded "gzip, deflate, br" while Brotli was+ not a dependency, so every server that honoured br returned a body aiohttp+ raised on. That was 852 of 978 capture errors across a 428-site run, with+ 173 sites failing completely.+ """+ from aliens_eye.core.config import DEFAULT_HEADERS++ encodings = {e.strip() for e in DEFAULT_HEADERS["Accept-Encoding"].split(",")}+ assert "gzip" in encodings and "deflate" in encodings++ try:+ import brotli # noqa: F401+ decoder = True+ except ImportError:+ try:+ import brotlicffi # noqa: F401+ decoder = True+ except ImportError:+ decoder = False++ assert ("br" in encodings) == decoder, (+ "Accept-Encoding advertises br without an importable Brotli decoder"+ if not decoder else+ "Brotli is installed but br is not advertised"+ )+++def test_brotli_is_a_declared_dependency():+ """It is required, not an extra: the default headers depend on it."""+ from pathlib import Path++ import tomllib++ root = Path(__file__).resolve().parents[1]+ data = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8"))+ deps = " ".join(data["project"]["dependencies"]).lower()+ assert "brotli" in deps