AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
fix: prevent bot-wall boilerplate from polluting correlation clusters
arxhr007ea1b9bdbbc9a
diff --git a/pyproject.toml b/pyproject.tomlindex 1b33d09..3b8f373 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "hatchling.build"[project]name = "aliens-eye"-version = "2.1.0"+version = "2.2.0"description = "AI-powered OSINT username scanner across 800+ social media and web platforms"readme = "README.md"requires-python = ">=3.10"diff --git a/src/aliens_eye/__init__.py b/src/aliens_eye/__init__.pyindex 29fb5be..01390a4 100644--- a/src/aliens_eye/__init__.py+++ b/src/aliens_eye/__init__.py@@ -1,3 +1,3 @@"""Aliens Eye - AI-powered OSINT username scanner."""-__version__ = "2.1.0"+__version__ = "2.2.0"diff --git a/src/aliens_eye/core/analyzer.py b/src/aliens_eye/core/analyzer.pyindex 6742190..2f416cd 100644--- a/src/aliens_eye/core/analyzer.py+++ b/src/aliens_eye/core/analyzer.py@@ -233,6 +233,10 @@ class FeatureExtractor:then plain <title>/<meta>. Per-site CSS overrides indata/site_profiles.json take precedence when present."""+ # A "structured" name comes from a real profile field (OpenGraph title,+ # JSON-LD Person, or a per-site CSS override). Only those are trustworthy+ # as an identity signal; the bare <title> fallback is page chrome (bot+ # walls, error pages, brand titles) and must not drive correlation.name = self._meta_content(tree, 'meta[property="og:title"]')bio = self._meta_content(tree, 'meta[property="og:description"]')avatar = self._meta_content(tree, 'meta[property="og:image"]')@@ -244,8 +248,6 @@ class FeatureExtractor:if not bio:bio = self._meta_content(tree, 'meta[name="description"]')- if not name:- name = title_textif not avatar:icon = tree.css_first('link[rel="apple-touch-icon"]')if icon:@@ -257,6 +259,10 @@ class FeatureExtractor:bio = self._select_text(tree, overrides.get("bio")) or bioavatar = self._select_attr(tree, overrides.get("avatar")) or avatar+ structured_name = (name or "").strip()+ if not name:+ name = title_text+if avatar:avatar = urljoin(url, avatar)@@ -264,6 +270,7 @@ class FeatureExtractor:"name": (name or "").strip()[:200],"bio": (bio or "").strip()[:500],"avatar": (avatar or "").strip(),+ "name_from_profile": bool(structured_name),}@staticmethoddiff --git a/src/aliens_eye/core/correlate.py b/src/aliens_eye/core/correlate.pyindex b7dccdf..c344490 100644--- a/src/aliens_eye/core/correlate.py+++ b/src/aliens_eye/core/correlate.py@@ -84,19 +84,28 @@ def _jaccard(a: set[str], b: set[str]) -> float:return inter / union if union else 0.0-def profiles_from_report(report: dict[str, Any]) -> list[Profile]:- """Extract Found/Maybe profiles (with any profile signals) from a report dict."""+def profiles_from_report(+ report: dict[str, Any], statuses: frozenset[str] = frozenset({"Found"}),+) -> list[Profile]:+ """Extract profiles (with profile signals) from a report dict.++ Defaults to Found-only: the Maybe band is dominated by 403/404 bot-walls whose+ boilerplate would pollute correlation. Only a *structured* display name (see+ ``name_from_profile``) is used as an identity signal — a bare ``<title>``+ fallback is dropped so page chrome cannot cluster unrelated sites.+ """profiles: list[Profile] = []for variation, block in report.get("variations", {}).items():for site, info in (block.get("sites", {}) or {}).items():- if info.get("status") not in {"Found", "Maybe"}:+ if info.get("status") not in statuses:continueprof = info.get("ai_analysis", {}).get("signals", {}).get("profile", {}) or {}bio = prof.get("bio", "") or ""+ identity_name = (prof.get("name", "") or "") if prof.get("name_from_profile") else ""p = Profile(variation=variation,site=site,- name=prof.get("name", "") or "",+ name=identity_name,bio=bio,avatar=prof.get("avatar", "") or "",url=info.get("url", "") or "",@@ -197,8 +206,15 @@ class _UnionFind:self.parent[rb] = ra-def _pair_reason(a: Profile, b: Profile) -> str | None:- """Return the reason two profiles are the same person, or None."""+def _pair_reason(+ a: Profile, b: Profile, common_names: frozenset[str] = frozenset()+) -> str | None:+ """Return the reason two profiles are the same person, or None.++ ``common_names`` holds normalized names that are too frequent across the scan+ to be identifying (boilerplate / generic brand titles); a name in that set+ never links a pair.+ """if (a.avatar_hash is not Noneand b.avatar_hash is not None@@ -210,21 +226,33 @@ def _pair_reason(a: Profile, b: Profile) -> str | None:if a.handles & b.handles or (a.variation and a.variation in b.handles) or (b.variation and b.variation in a.handles):return "handle"na, nb = _norm_name(a.name), _norm_name(b.name)- if na and na == nb and len(na) > 3:+ if na and na == nb and len(na) > 3 and na not in common_names:return "name"if _jaccard(_tokens(a.bio), _tokens(b.bio)) >= BIO_MIN_JACCARD:return "bio"return None+def _common_names(profiles: list[Profile]) -> frozenset[str]:+ """Normalized names shared by too many profiles to be identifying."""+ from collections import Counter++ counts: Counter[str] = Counter(+ na for p in profiles if (na := _norm_name(p.name))+ )+ threshold = max(4, int(0.02 * len(profiles)))+ return frozenset(name for name, count in counts.items() if count > threshold)++def cluster_profiles(profiles: list[Profile]) -> list[dict[str, Any]]:"""Union-find clustering. Returns clusters as dicts (only multi-site clusters)."""n = len(profiles)uf = _UnionFind(n)+ common_names = _common_names(profiles)reasons: dict[tuple[int, int], str] = {}for i in range(n):for j in range(i + 1, n):- reason = _pair_reason(profiles[i], profiles[j])+ reason = _pair_reason(profiles[i], profiles[j], common_names)if reason:uf.union(i, j)reasons[(i, j)] = reasondiff --git a/tests/test_new_features.py b/tests/test_new_features.pyindex 86c9d6e..b31b99a 100644--- a/tests/test_new_features.py+++ b/tests/test_new_features.py@@ -17,7 +17,10 @@ from aliens_eye.ml.collect import write_rowsfrom aliens_eye.selfcheck import _metrics-def _profile_result(site, variation, status="Found", name="", bio="", avatar="", url=""):+def _profile_result(+ site, variation, status="Found", name="", bio="", avatar="", url="",+ name_from_profile=True,+):return {"site": site,"url": url or f"https://{site}.com/{variation}",@@ -28,7 +31,14 @@ def _profile_result(site, variation, status="Found", name="", bio="", avatar="","confidence": 90,"ai_analysis": {"features": {"http_200": 1.0},- "signals": {"profile": {"name": name, "bio": bio, "avatar": avatar}},+ "signals": {+ "profile": {+ "name": name,+ "bio": bio,+ "avatar": avatar,+ "name_from_profile": name_from_profile,+ }+ },},}@@ -115,7 +125,7 @@ def test_no_cluster_when_unrelated():def test_profiles_from_report_skips_not_found():rep = _report({"alice": [- _profile_result("github", "alice", status="Found", name="A"),+ _profile_result("github", "alice", status="Found", name="Alice Smith"),_profile_result("reddit", "alice", status="Not Found"),]})@@ -123,6 +133,47 @@ def test_profiles_from_report_skips_not_found():assert [p.site for p in profiles] == ["github"]+def test_profiles_from_report_is_found_only():+ # Maybe-band results (mostly bot-walls) must be excluded by default.+ rep = _report({+ "alice": [+ _profile_result("github", "alice", status="Found", name="Alice Smith"),+ _profile_result("reddit", "alice", status="Maybe", name="Alice Smith"),+ ]+ })+ profiles = profiles_from_report(rep)+ assert [p.site for p in profiles] == ["github"]+++def test_title_derived_name_not_used_for_identity():+ # Same name on two sites, but it came from <title> (name_from_profile=False):+ # must NOT cluster. Flip the flag on and it should.+ junk = _report({+ "alice": [+ _profile_result("s1", "alice", name="Just a moment...", name_from_profile=False),+ _profile_result("s2", "alice", name="Just a moment...", name_from_profile=False),+ ]+ })+ assert cluster_profiles(profiles_from_report(junk)) == []++ real = _report({+ "alice": [+ _profile_result("s1", "alice", name="Alice Smith", name_from_profile=True),+ _profile_result("s2", "alice", name="Alice Smith", name_from_profile=True),+ ]+ })+ assert len(cluster_profiles(profiles_from_report(real))) == 1+++def test_common_name_downweighted():+ # A structured name shared by many profiles is boilerplate -> no cluster.+ profiles = [+ Profile("u", f"s{i}", name="Sign In", bio="", avatar="", url="", status="Found")+ for i in range(10)+ ]+ assert cluster_profiles(profiles) == []++# --- expansion ------------------------------------------------------------def test_candidate_usernames_from_bio():