Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

fix: percent-encode usernames and penalize redirect-away interstitials (#17)

arxhr0071ca08daa1cc7
diff --git a/pyproject.toml b/pyproject.toml
index 3b8f373..82a1c9c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "aliens-eye"
-version = "2.2.0"
+version = "2.2.1"
description = "AI-powered OSINT username scanner across 800+ social media and web platforms"
readme = "README.md"
requires-python = ">=3.10"
diff --git a/src/aliens_eye/__init__.py b/src/aliens_eye/__init__.py
index 01390a4..6d1dba8 100644
--- a/src/aliens_eye/__init__.py
+++ b/src/aliens_eye/__init__.py
@@ -1,3 +1,3 @@
"""Aliens Eye - AI-powered OSINT username scanner."""
-__version__ = "2.2.0"
+__version__ = "2.2.1"
diff --git a/src/aliens_eye/core/detector.py b/src/aliens_eye/core/detector.py
index d9b2e50..f5a568c 100644
--- a/src/aliens_eye/core/detector.py
+++ b/src/aliens_eye/core/detector.py
@@ -92,6 +92,12 @@ class Detector:
if features.get("is_homepage", 0.0) > 0 and features.get("http_200", 0.0) > 0:
score -= 5
+ # Redirected off a username-bearing URL to one that no longer has it: the
+ # structural signature of a visitor/login/geo/bot-wall interstitial, not a
+ # profile — regardless of language or the specific wording used.
+ if features.get("redirect_count", 0.0) > 0 and features.get("has_username_in_path", 0.0) == 0:
+ score -= 4
+
score -= features.get("error_section_count", 0.0) * 3
score += features.get("profile_section_count", 0.0) * 4
diff --git a/src/aliens_eye/core/scanner.py b/src/aliens_eye/core/scanner.py
index ee98ba8..a05fc14 100644
--- a/src/aliens_eye/core/scanner.py
+++ b/src/aliens_eye/core/scanner.py
@@ -5,6 +5,7 @@ from dataclasses import dataclass
from importlib import resources
from pathlib import Path
from typing import Any
+from urllib.parse import quote
import aiohttp
@@ -99,6 +100,26 @@ def filter_sites(
return result
+def format_site_url(site_name: str, url_template: str, username: str) -> str:
+ """Substitute a username into a site's URL template, percent-encoded.
+
+ ``quote()`` leaves ASCII letters/digits/``-_.~`` untouched, so ordinary
+ handles are unaffected; non-ASCII (CJK, etc.) and otherwise URL-unsafe
+ characters (spaces, ``@``, ``#``, ...) are percent-encoded rather than
+ pasted into the URL verbatim.
+ """
+ encoded = quote(username, safe="")
+ try:
+ url = url_template.format(encoded)
+ except (KeyError, IndexError):
+ url = url_template.replace("{}", encoded)
+ if "{" in url:
+ url = f"https://{site_name}.com/{encoded}"
+ except Exception:
+ url = f"https://{site_name}.com/{encoded}"
+ return url
+
+
def build_connector(config: ScannerConfig, limit: int) -> aiohttp.BaseConnector:
"""TCP connector, or a SOCKS proxy connector when a socks:// proxy is set."""
if config.proxy and config.proxy.lower().startswith(("socks4://", "socks5://")):
@@ -150,6 +171,13 @@ class UsernameScanner:
more = f" and {len(variations) - 5} more..." if len(variations) > 5 else ""
self.console.print(f"[dim]Variations: {preview}{more}[/dim]")
+ if not base_username.isascii():
+ self.console.print(
+ "[dim]Note: non-ASCII username. URLs are percent-encoded, but many "
+ "platforms key profiles by an internal ID rather than the display "
+ "handle, so results on those sites may be unreliable.[/dim]"
+ )
+
for username in variations:
results = await self.scan_all_sites(username)
all_results[username] = results
@@ -394,12 +422,4 @@ class UsernameScanner:
@staticmethod
def _format_url(site_name: str, url_template: str, username: str) -> str:
- try:
- url = url_template.format(username)
- except (KeyError, IndexError):
- url = url_template.replace("{}", username)
- if "{" in url:
- url = f"https://{site_name}.com/{username}"
- except Exception:
- url = f"https://{site_name}.com/{username}"
- return url
+ return format_site_url(site_name, url_template, username)
diff --git a/src/aliens_eye/ml/collect.py b/src/aliens_eye/ml/collect.py
index cec2feb..1fbbfd8 100644
--- a/src/aliens_eye/ml/collect.py
+++ b/src/aliens_eye/ml/collect.py
@@ -24,6 +24,7 @@ from aliens_eye.core.detector import Detector
from aliens_eye.core.features import FEATURE_SCHEMA, vectorize_features
from aliens_eye.core.http import fetch_url
from aliens_eye.core.rate_limit import DomainRateLimiter
+from aliens_eye.core.scanner import format_site_url
def load_selfcheck_data() -> dict[str, list[str]]:
@@ -65,10 +66,7 @@ async def _collect_row(
label: int,
logger,
) -> list[float] | None:
- try:
- url = url_template.format(username)
- except Exception:
- url = url_template.replace("{}", username)
+ url = format_site_url(site, url_template, username)
fetch = await fetch_url(session, url, config, rate_limiter, logger)
if fetch.error:
return None
diff --git a/src/aliens_eye/selfcheck.py b/src/aliens_eye/selfcheck.py
index fd7b537..3598ed0 100644
--- a/src/aliens_eye/selfcheck.py
+++ b/src/aliens_eye/selfcheck.py
@@ -25,6 +25,7 @@ from aliens_eye.core.config import DEFAULT_HEADERS, ScannerConfig
from aliens_eye.core.detector import Detector
from aliens_eye.core.http import fetch_url
from aliens_eye.core.rate_limit import DomainRateLimiter
+from aliens_eye.core.scanner import format_site_url
from aliens_eye.utils.console import get_console
@@ -97,10 +98,7 @@ async def run_selfcheck(
async with aiohttp.ClientSession(headers=DEFAULT_HEADERS, connector=connector) as session:
async def check(site: str, template: str, username: str, label: int) -> None:
- try:
- url = template.format(username)
- except Exception:
- url = template.replace("{}", username)
+ url = format_site_url(site, template, username)
async with semaphore:
fetch = await fetch_url(session, url, config, rate_limiter, logger)
if fetch.error:
diff --git a/tests/test_detector.py b/tests/test_detector.py
index fbed3f2..8117d8a 100644
--- a/tests/test_detector.py
+++ b/tests/test_detector.py
@@ -90,6 +90,26 @@ def test_load_packaged_model_blends(logger):
assert 0.0 <= result.probability <= 1.0
+def test_redirect_off_username_path_penalized():
+ # Same base features, but one landed on a generic page after a redirect
+ # (visitor/login/geo-block-wall pattern) -> should score lower.
+ detector = Detector()
+ base = {"http_200": 1.0, "positive_keyword_count": 3.0}
+ on_profile = {**base, "has_username_in_path": 1.0, "redirect_count": 1.0}
+ bounced = {**base, "has_username_in_path": 0.0, "redirect_count": 1.0}
+ assert detector.predict(on_profile).score > detector.predict(bounced).score
+
+
+def test_redirect_without_leaving_username_path_not_penalized():
+ # A redirect that still lands on a URL containing the username (e.g. a
+ # canonical-URL normalization) should not be penalized.
+ detector = Detector()
+ base = {"http_200": 1.0, "has_username_in_path": 1.0}
+ assert detector.predict({**base, "redirect_count": 0.0}).score == detector.predict(
+ {**base, "redirect_count": 1.0}
+ ).score
+
+
def test_sigmoid_bounds():
assert _sigmoid(1000) == 1.0
assert _sigmoid(-1000) < 1e-20
diff --git a/tests/test_scanner.py b/tests/test_scanner.py
index 93396b6..ce541ee 100644
--- a/tests/test_scanner.py
+++ b/tests/test_scanner.py
@@ -9,7 +9,7 @@ from aliens_eye.core.analyzer import FeatureExtractor
from aliens_eye.core.config import ScannerConfig
from aliens_eye.core.detector import Detector
from aliens_eye.core.fingerprints import FingerprintStore
-from aliens_eye.core.scanner import UsernameScanner, build_connector
+from aliens_eye.core.scanner import UsernameScanner, build_connector, format_site_url
def free_port() -> int:
@@ -105,6 +105,21 @@ def test_format_url_fallbacks():
)
+def test_format_site_url_percent_encodes_non_ascii():
+ url = format_site_url("weibo", "https://weibo.com/{}", "坦途")
+ assert url == "https://weibo.com/%E5%9D%A6%E9%80%94"
+ assert "坦途" not in url
+
+
+def test_format_site_url_leaves_ascii_handles_unchanged():
+ assert format_site_url("x", "https://x.com/{}", "bob_dev-99") == "https://x.com/bob_dev-99"
+
+
+def test_format_site_url_encodes_special_characters():
+ url = format_site_url("x", "https://x.com/{}", "a b@c")
+ assert url == "https://x.com/a%20b%40c"
+
+
async def test_build_connector_tcp():
config = ScannerConfig()
connector = build_connector(config, 5)