Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

fix: declare Brotli and advertise only decodable content encodings

arxhr007401f0e2cfdab
diff --git a/pyproject.toml b/pyproject.toml
index c51c822..b14c054 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,6 +26,9 @@ classifiers = [
]
dependencies = [
"aiohttp>=3.8",
+ # Required, not optional: DEFAULT_HEADERS advertises br, and a server that
+ # honours it returns a body aiohttp raises on without a Brotli decoder.
+ "Brotli>=1.1",
"selectolax>=0.3.21",
"rich>=13.7",
"aiohttp-socks>=0.8,<1.0",
diff --git a/src/aliens_eye/core/config.py b/src/aliens_eye/core/config.py
index 03e1ea0..368304f 100644
--- a/src/aliens_eye/core/config.py
+++ b/src/aliens_eye/core/config.py
@@ -10,10 +10,32 @@ DEFAULT_USER_AGENT = (
"AppleWebKit/537.36 Chrome/115.0.0.0 Safari/537.36"
)
+def _supported_encodings() -> str:
+ """Advertise only what this install can actually decode.
+
+ Claiming ``br`` without a Brotli decoder present is not a degraded response,
+ it is a hard failure: any server that honours it returns a body aiohttp
+ raises on ("can not decode content-encoding: brotli"), so the site is scanned
+ as an error rather than checked. Brotli is a declared dependency, but an
+ existing environment may predate that, so the header is built from what is
+ importable rather than assumed.
+ """
+ encodings = ["gzip", "deflate"]
+ try: # pragma: no cover - exercised by whichever branch the env provides
+ import brotli # noqa: F401
+ except ImportError:
+ try:
+ import brotlicffi # noqa: F401
+ except ImportError:
+ return ", ".join(encodings)
+ encodings.append("br")
+ return ", ".join(encodings)
+
+
DEFAULT_HEADERS = {
"User-Agent": DEFAULT_USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
- "Accept-Encoding": "gzip, deflate, br",
+ "Accept-Encoding": _supported_encodings(),
"Accept-Language": "en-US,en;q=0.5",
"Connection": "keep-alive",
"DNT": "1",
diff --git a/tests/test_config_cli.py b/tests/test_config_cli.py
index 524c87a..7a484b7 100644
--- a/tests/test_config_cli.py
+++ b/tests/test_config_cli.py
@@ -98,3 +98,45 @@ def test_load_config_explicit_path(tmp_path, logger):
def test_load_config_missing_returns_empty(tmp_path, logger):
assert load_config(str(tmp_path / "nope.json"), logger) == {}
+
+
+def test_accept_encoding_only_advertises_decodable_encodings():
+ """Advertising br without a decoder is a hard failure, not a degradation.
+
+ Regression: DEFAULT_HEADERS hardcoded "gzip, deflate, br" while Brotli was
+ not a dependency, so every server that honoured br returned a body aiohttp
+ raised on. That was 852 of 978 capture errors across a 428-site run, with
+ 173 sites failing completely.
+ """
+ from aliens_eye.core.config import DEFAULT_HEADERS
+
+ encodings = {e.strip() for e in DEFAULT_HEADERS["Accept-Encoding"].split(",")}
+ assert "gzip" in encodings and "deflate" in encodings
+
+ try:
+ import brotli # noqa: F401
+ decoder = True
+ except ImportError:
+ try:
+ import brotlicffi # noqa: F401
+ decoder = True
+ except ImportError:
+ decoder = False
+
+ assert ("br" in encodings) == decoder, (
+ "Accept-Encoding advertises br without an importable Brotli decoder"
+ if not decoder else
+ "Brotli is installed but br is not advertised"
+ )
+
+
+def test_brotli_is_a_declared_dependency():
+ """It is required, not an extra: the default headers depend on it."""
+ from pathlib import Path
+
+ import tomllib
+
+ root = Path(__file__).resolve().parents[1]
+ data = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8"))
+ deps = " ".join(data["project"]["dependencies"]).lower()
+ assert "brotli" in deps