Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

fix: give every site's check an overall deadline

arxhr007a1fc7bb8aab0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d33fe6b..d8780ca 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,9 @@
for an hour (and the wait was applied twice). The header is the server's to set, so it is
now honoured only up to the retry backoff cap (8 seconds); a site asking for longer is
reported as an error and not asked again.
+- Every site's check now has an overall deadline (about 80 seconds with default settings),
+ after which it is reported as `Timeout`. A scan is only as fast as its slowest site, and
+ a check that never returned kept the whole scan open.
- Cancelling a scan from code that uses the API (for example with a timeout) left its
worker tasks running.
diff --git a/src/aliens_eye/core/scanner.py b/src/aliens_eye/core/scanner.py
index b63f640..83995d9 100644
--- a/src/aliens_eye/core/scanner.py
+++ b/src/aliens_eye/core/scanner.py
@@ -329,10 +329,25 @@ class UsernameScanner:
except asyncio.CancelledError:
break
try:
- result = await self._scan_site(
- site, tmpl, username, session, rate_limiter
+ result = await asyncio.wait_for(
+ self._scan_site(site, tmpl, username, session, rate_limiter),
+ self._site_deadline(),
)
results.append(result)
+ except asyncio.TimeoutError:
+ url = self._format_url(site, tmpl, username)
+ self.logger.debug("Gave up on %s after %.0fs", site, self._site_deadline())
+ result = {
+ "site": site,
+ "url": url,
+ "final_url": url,
+ "status": "Timeout",
+ "code": 408,
+ "response_time": 0.0,
+ "confidence": 0,
+ "ai_analysis": {"error": "no answer within the per-site deadline"},
+ }
+ results.append(result)
except Exception as exc:
url = self._format_url(site, tmpl, username)
self.logger.debug("Worker failed for %s: %s", site, exc)
@@ -363,6 +378,20 @@ class UsernameScanner:
view.advance(found)
queue.task_done()
+ def _site_deadline(self) -> float:
+ """Longest one site's check may take, whatever goes wrong inside it.
+
+ Every attempt has its own timeout and every wait between attempts is
+ capped, so a healthy check ends well inside this. It exists for the
+ cases those bounds miss: a scan is only as fast as its slowest site, and
+ one that never returns holds the whole scan open.
+ """
+ config = self.config
+ budget = (config.retries + 1) * (config.timeout + config.backoff_cap + config.jitter)
+ if config.use_playwright and self.browser_fallback is not None:
+ budget += 60.0
+ return budget * 1.5 + 1.0
+
async def _scan_site(
self,
site_name: str,
diff --git a/tests/test_scanner.py b/tests/test_scanner.py
index 8b61d5f..244db87 100644
--- a/tests/test_scanner.py
+++ b/tests/test_scanner.py
@@ -230,3 +230,26 @@ async def test_a_cancelled_scan_leaves_no_workers_behind(tmp_path, logger):
assert left == []
finally:
await server.close()
+
+
+async def test_one_site_that_never_answers_cannot_hold_up_the_scan(site_server, tmp_path, logger):
+ """A real scan sat on its last request for half an hour. Whatever the cause
+ inside one site's check, the scan moves on after a bounded wait."""
+ import asyncio
+
+ from aliens_eye.core.http import fetch_url
+
+ scanner = make_scanner(server_sites(site_server), tmp_path, logger)
+ scanner.config.timeout = 0.2
+ scanner.config.backoff_cap = 0.0
+
+ async def fetch(session, url, config, rate_limiter, log):
+ if "/beta/" in url:
+ await asyncio.sleep(3600) # ignores every timeout the HTTP layer has
+ return await fetch_url(session, url, config, rate_limiter, log)
+
+ scanner.fetch = fetch
+ results = await asyncio.wait_for(scanner.scan_all_sites("torvalds"), 10)
+ by_site = {r["site"]: r for r in results}
+ assert by_site["alpha"]["status"] == "Found"
+ assert by_site["beta"]["status"] == "Timeout" and by_site["beta"]["code"] == 408