AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
fix: percent-encode usernames and penalize redirect-away interstitials (#17)
arxhr0071ca08daa1cc7
diff --git a/pyproject.toml b/pyproject.tomlindex 3b8f373..82a1c9c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "hatchling.build"[project]name = "aliens-eye"-version = "2.2.0"+version = "2.2.1"description = "AI-powered OSINT username scanner across 800+ social media and web platforms"readme = "README.md"requires-python = ">=3.10"diff --git a/src/aliens_eye/__init__.py b/src/aliens_eye/__init__.pyindex 01390a4..6d1dba8 100644--- a/src/aliens_eye/__init__.py+++ b/src/aliens_eye/__init__.py@@ -1,3 +1,3 @@"""Aliens Eye - AI-powered OSINT username scanner."""-__version__ = "2.2.0"+__version__ = "2.2.1"diff --git a/src/aliens_eye/core/detector.py b/src/aliens_eye/core/detector.pyindex d9b2e50..f5a568c 100644--- a/src/aliens_eye/core/detector.py+++ b/src/aliens_eye/core/detector.py@@ -92,6 +92,12 @@ class Detector:if features.get("is_homepage", 0.0) > 0 and features.get("http_200", 0.0) > 0:score -= 5+ # Redirected off a username-bearing URL to one that no longer has it: the+ # structural signature of a visitor/login/geo/bot-wall interstitial, not a+ # profile — regardless of language or the specific wording used.+ if features.get("redirect_count", 0.0) > 0 and features.get("has_username_in_path", 0.0) == 0:+ score -= 4+score -= features.get("error_section_count", 0.0) * 3score += features.get("profile_section_count", 0.0) * 4diff --git a/src/aliens_eye/core/scanner.py b/src/aliens_eye/core/scanner.pyindex ee98ba8..a05fc14 100644--- a/src/aliens_eye/core/scanner.py+++ b/src/aliens_eye/core/scanner.py@@ -5,6 +5,7 @@ from dataclasses import dataclassfrom importlib import resourcesfrom pathlib import Pathfrom typing import Any+from urllib.parse import quoteimport aiohttp@@ -99,6 +100,26 @@ def filter_sites(return result+def format_site_url(site_name: str, url_template: str, username: str) -> str:+ """Substitute a username into a site's URL template, percent-encoded.++ ``quote()`` leaves ASCII letters/digits/``-_.~`` untouched, so ordinary+ handles are unaffected; non-ASCII (CJK, etc.) and otherwise URL-unsafe+ characters (spaces, ``@``, ``#``, ...) are percent-encoded rather than+ pasted into the URL verbatim.+ """+ encoded = quote(username, safe="")+ try:+ url = url_template.format(encoded)+ except (KeyError, IndexError):+ url = url_template.replace("{}", encoded)+ if "{" in url:+ url = f"https://{site_name}.com/{encoded}"+ except Exception:+ url = f"https://{site_name}.com/{encoded}"+ return url++def build_connector(config: ScannerConfig, limit: int) -> aiohttp.BaseConnector:"""TCP connector, or a SOCKS proxy connector when a socks:// proxy is set."""if config.proxy and config.proxy.lower().startswith(("socks4://", "socks5://")):@@ -150,6 +171,13 @@ class UsernameScanner:more = f" and {len(variations) - 5} more..." if len(variations) > 5 else ""self.console.print(f"[dim]Variations: {preview}{more}[/dim]")+ if not base_username.isascii():+ self.console.print(+ "[dim]Note: non-ASCII username. URLs are percent-encoded, but many "+ "platforms key profiles by an internal ID rather than the display "+ "handle, so results on those sites may be unreliable.[/dim]"+ )+for username in variations:results = await self.scan_all_sites(username)all_results[username] = results@@ -394,12 +422,4 @@ class UsernameScanner:@staticmethoddef _format_url(site_name: str, url_template: str, username: str) -> str:- try:- url = url_template.format(username)- except (KeyError, IndexError):- url = url_template.replace("{}", username)- if "{" in url:- url = f"https://{site_name}.com/{username}"- except Exception:- url = f"https://{site_name}.com/{username}"- return url+ return format_site_url(site_name, url_template, username)diff --git a/src/aliens_eye/ml/collect.py b/src/aliens_eye/ml/collect.pyindex cec2feb..1fbbfd8 100644--- a/src/aliens_eye/ml/collect.py+++ b/src/aliens_eye/ml/collect.py@@ -24,6 +24,7 @@ from aliens_eye.core.detector import Detectorfrom aliens_eye.core.features import FEATURE_SCHEMA, vectorize_featuresfrom aliens_eye.core.http import fetch_urlfrom aliens_eye.core.rate_limit import DomainRateLimiter+from aliens_eye.core.scanner import format_site_urldef load_selfcheck_data() -> dict[str, list[str]]:@@ -65,10 +66,7 @@ async def _collect_row(label: int,logger,) -> list[float] | None:- try:- url = url_template.format(username)- except Exception:- url = url_template.replace("{}", username)+ url = format_site_url(site, url_template, username)fetch = await fetch_url(session, url, config, rate_limiter, logger)if fetch.error:return Nonediff --git a/src/aliens_eye/selfcheck.py b/src/aliens_eye/selfcheck.pyindex fd7b537..3598ed0 100644--- a/src/aliens_eye/selfcheck.py+++ b/src/aliens_eye/selfcheck.py@@ -25,6 +25,7 @@ from aliens_eye.core.config import DEFAULT_HEADERS, ScannerConfigfrom aliens_eye.core.detector import Detectorfrom aliens_eye.core.http import fetch_urlfrom aliens_eye.core.rate_limit import DomainRateLimiter+from aliens_eye.core.scanner import format_site_urlfrom aliens_eye.utils.console import get_console@@ -97,10 +98,7 @@ async def run_selfcheck(async with aiohttp.ClientSession(headers=DEFAULT_HEADERS, connector=connector) as session:async def check(site: str, template: str, username: str, label: int) -> None:- try:- url = template.format(username)- except Exception:- url = template.replace("{}", username)+ url = format_site_url(site, template, username)async with semaphore:fetch = await fetch_url(session, url, config, rate_limiter, logger)if fetch.error:diff --git a/tests/test_detector.py b/tests/test_detector.pyindex fbed3f2..8117d8a 100644--- a/tests/test_detector.py+++ b/tests/test_detector.py@@ -90,6 +90,26 @@ def test_load_packaged_model_blends(logger):assert 0.0 <= result.probability <= 1.0+def test_redirect_off_username_path_penalized():+ # Same base features, but one landed on a generic page after a redirect+ # (visitor/login/geo-block-wall pattern) -> should score lower.+ detector = Detector()+ base = {"http_200": 1.0, "positive_keyword_count": 3.0}+ on_profile = {**base, "has_username_in_path": 1.0, "redirect_count": 1.0}+ bounced = {**base, "has_username_in_path": 0.0, "redirect_count": 1.0}+ assert detector.predict(on_profile).score > detector.predict(bounced).score+++def test_redirect_without_leaving_username_path_not_penalized():+ # A redirect that still lands on a URL containing the username (e.g. a+ # canonical-URL normalization) should not be penalized.+ detector = Detector()+ base = {"http_200": 1.0, "has_username_in_path": 1.0}+ assert detector.predict({**base, "redirect_count": 0.0}).score == detector.predict(+ {**base, "redirect_count": 1.0}+ ).score++def test_sigmoid_bounds():assert _sigmoid(1000) == 1.0assert _sigmoid(-1000) < 1e-20diff --git a/tests/test_scanner.py b/tests/test_scanner.pyindex 93396b6..ce541ee 100644--- a/tests/test_scanner.py+++ b/tests/test_scanner.py@@ -9,7 +9,7 @@ from aliens_eye.core.analyzer import FeatureExtractorfrom aliens_eye.core.config import ScannerConfigfrom aliens_eye.core.detector import Detectorfrom aliens_eye.core.fingerprints import FingerprintStore-from aliens_eye.core.scanner import UsernameScanner, build_connector+from aliens_eye.core.scanner import UsernameScanner, build_connector, format_site_urldef free_port() -> int:@@ -105,6 +105,21 @@ def test_format_url_fallbacks():)+def test_format_site_url_percent_encodes_non_ascii():+ url = format_site_url("weibo", "https://weibo.com/{}", "坦途")+ assert url == "https://weibo.com/%E5%9D%A6%E9%80%94"+ assert "坦途" not in url+++def test_format_site_url_leaves_ascii_handles_unchanged():+ assert format_site_url("x", "https://x.com/{}", "bob_dev-99") == "https://x.com/bob_dev-99"+++def test_format_site_url_encodes_special_characters():+ url = format_site_url("x", "https://x.com/{}", "a b@c")+ assert url == "https://x.com/a%20b%40c"++async def test_build_connector_tcp():config = ScannerConfig()connector = build_connector(config, 5)