AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
fix: give every site's check an overall deadline
arxhr007a1fc7bb8aab0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex d33fe6b..d8780ca 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,9 @@for an hour (and the wait was applied twice). The header is the server's to set, so it isnow honoured only up to the retry backoff cap (8 seconds); a site asking for longer isreported as an error and not asked again.+- Every site's check now has an overall deadline (about 80 seconds with default settings),+ after which it is reported as `Timeout`. A scan is only as fast as its slowest site, and+ a check that never returned kept the whole scan open.- Cancelling a scan from code that uses the API (for example with a timeout) left itsworker tasks running.diff --git a/src/aliens_eye/core/scanner.py b/src/aliens_eye/core/scanner.pyindex b63f640..83995d9 100644--- a/src/aliens_eye/core/scanner.py+++ b/src/aliens_eye/core/scanner.py@@ -329,10 +329,25 @@ class UsernameScanner:except asyncio.CancelledError:breaktry:- result = await self._scan_site(- site, tmpl, username, session, rate_limiter+ result = await asyncio.wait_for(+ self._scan_site(site, tmpl, username, session, rate_limiter),+ self._site_deadline(),)results.append(result)+ except asyncio.TimeoutError:+ url = self._format_url(site, tmpl, username)+ self.logger.debug("Gave up on %s after %.0fs", site, self._site_deadline())+ result = {+ "site": site,+ "url": url,+ "final_url": url,+ "status": "Timeout",+ "code": 408,+ "response_time": 0.0,+ "confidence": 0,+ "ai_analysis": {"error": "no answer within the per-site deadline"},+ }+ results.append(result)except Exception as exc:url = self._format_url(site, tmpl, username)self.logger.debug("Worker failed for %s: %s", site, exc)@@ -363,6 +378,20 @@ class UsernameScanner:view.advance(found)queue.task_done()+ def _site_deadline(self) -> float:+ """Longest one site's check may take, whatever goes wrong inside it.++ Every attempt has its own timeout and every wait between attempts is+ capped, so a healthy check ends well inside this. It exists for the+ cases those bounds miss: a scan is only as fast as its slowest site, and+ one that never returns holds the whole scan open.+ """+ config = self.config+ budget = (config.retries + 1) * (config.timeout + config.backoff_cap + config.jitter)+ if config.use_playwright and self.browser_fallback is not None:+ budget += 60.0+ return budget * 1.5 + 1.0+async def _scan_site(self,site_name: str,diff --git a/tests/test_scanner.py b/tests/test_scanner.pyindex 8b61d5f..244db87 100644--- a/tests/test_scanner.py+++ b/tests/test_scanner.py@@ -230,3 +230,26 @@ async def test_a_cancelled_scan_leaves_no_workers_behind(tmp_path, logger):assert left == []finally:await server.close()+++async def test_one_site_that_never_answers_cannot_hold_up_the_scan(site_server, tmp_path, logger):+ """A real scan sat on its last request for half an hour. Whatever the cause+ inside one site's check, the scan moves on after a bounded wait."""+ import asyncio++ from aliens_eye.core.http import fetch_url++ scanner = make_scanner(server_sites(site_server), tmp_path, logger)+ scanner.config.timeout = 0.2+ scanner.config.backoff_cap = 0.0++ async def fetch(session, url, config, rate_limiter, log):+ if "/beta/" in url:+ await asyncio.sleep(3600) # ignores every timeout the HTTP layer has+ return await fetch_url(session, url, config, rate_limiter, log)++ scanner.fetch = fetch+ results = await asyncio.wait_for(scanner.scan_all_sites("torvalds"), 10)+ by_site = {r["site"]: r for r in results}+ assert by_site["alpha"]["status"] == "Found"+ assert by_site["beta"]["status"] == "Timeout" and by_site["beta"]["code"] == 408