Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

fix: prevent bot-wall boilerplate from polluting correlation clusters

arxhr007ea1b9bdbbc9a
diff --git a/pyproject.toml b/pyproject.toml
index 1b33d09..3b8f373 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "aliens-eye"
-version = "2.1.0"
+version = "2.2.0"
description = "AI-powered OSINT username scanner across 800+ social media and web platforms"
readme = "README.md"
requires-python = ">=3.10"
diff --git a/src/aliens_eye/__init__.py b/src/aliens_eye/__init__.py
index 29fb5be..01390a4 100644
--- a/src/aliens_eye/__init__.py
+++ b/src/aliens_eye/__init__.py
@@ -1,3 +1,3 @@
"""Aliens Eye - AI-powered OSINT username scanner."""
-__version__ = "2.1.0"
+__version__ = "2.2.0"
diff --git a/src/aliens_eye/core/analyzer.py b/src/aliens_eye/core/analyzer.py
index 6742190..2f416cd 100644
--- a/src/aliens_eye/core/analyzer.py
+++ b/src/aliens_eye/core/analyzer.py
@@ -233,6 +233,10 @@ class FeatureExtractor:
then plain <title>/<meta>. Per-site CSS overrides in
data/site_profiles.json take precedence when present.
"""
+ # A "structured" name comes from a real profile field (OpenGraph title,
+ # JSON-LD Person, or a per-site CSS override). Only those are trustworthy
+ # as an identity signal; the bare <title> fallback is page chrome (bot
+ # walls, error pages, brand titles) and must not drive correlation.
name = self._meta_content(tree, 'meta[property="og:title"]')
bio = self._meta_content(tree, 'meta[property="og:description"]')
avatar = self._meta_content(tree, 'meta[property="og:image"]')
@@ -244,8 +248,6 @@ class FeatureExtractor:
if not bio:
bio = self._meta_content(tree, 'meta[name="description"]')
- if not name:
- name = title_text
if not avatar:
icon = tree.css_first('link[rel="apple-touch-icon"]')
if icon:
@@ -257,6 +259,10 @@ class FeatureExtractor:
bio = self._select_text(tree, overrides.get("bio")) or bio
avatar = self._select_attr(tree, overrides.get("avatar")) or avatar
+ structured_name = (name or "").strip()
+ if not name:
+ name = title_text
+
if avatar:
avatar = urljoin(url, avatar)
@@ -264,6 +270,7 @@ class FeatureExtractor:
"name": (name or "").strip()[:200],
"bio": (bio or "").strip()[:500],
"avatar": (avatar or "").strip(),
+ "name_from_profile": bool(structured_name),
}
@staticmethod
diff --git a/src/aliens_eye/core/correlate.py b/src/aliens_eye/core/correlate.py
index b7dccdf..c344490 100644
--- a/src/aliens_eye/core/correlate.py
+++ b/src/aliens_eye/core/correlate.py
@@ -84,19 +84,28 @@ def _jaccard(a: set[str], b: set[str]) -> float:
return inter / union if union else 0.0
-def profiles_from_report(report: dict[str, Any]) -> list[Profile]:
- """Extract Found/Maybe profiles (with any profile signals) from a report dict."""
+def profiles_from_report(
+ report: dict[str, Any], statuses: frozenset[str] = frozenset({"Found"}),
+) -> list[Profile]:
+ """Extract profiles (with profile signals) from a report dict.
+
+ Defaults to Found-only: the Maybe band is dominated by 403/404 bot-walls whose
+ boilerplate would pollute correlation. Only a *structured* display name (see
+ ``name_from_profile``) is used as an identity signal — a bare ``<title>``
+ fallback is dropped so page chrome cannot cluster unrelated sites.
+ """
profiles: list[Profile] = []
for variation, block in report.get("variations", {}).items():
for site, info in (block.get("sites", {}) or {}).items():
- if info.get("status") not in {"Found", "Maybe"}:
+ if info.get("status") not in statuses:
continue
prof = info.get("ai_analysis", {}).get("signals", {}).get("profile", {}) or {}
bio = prof.get("bio", "") or ""
+ identity_name = (prof.get("name", "") or "") if prof.get("name_from_profile") else ""
p = Profile(
variation=variation,
site=site,
- name=prof.get("name", "") or "",
+ name=identity_name,
bio=bio,
avatar=prof.get("avatar", "") or "",
url=info.get("url", "") or "",
@@ -197,8 +206,15 @@ class _UnionFind:
self.parent[rb] = ra
-def _pair_reason(a: Profile, b: Profile) -> str | None:
- """Return the reason two profiles are the same person, or None."""
+def _pair_reason(
+ a: Profile, b: Profile, common_names: frozenset[str] = frozenset()
+) -> str | None:
+ """Return the reason two profiles are the same person, or None.
+
+ ``common_names`` holds normalized names that are too frequent across the scan
+ to be identifying (boilerplate / generic brand titles); a name in that set
+ never links a pair.
+ """
if (
a.avatar_hash is not None
and b.avatar_hash is not None
@@ -210,21 +226,33 @@ def _pair_reason(a: Profile, b: Profile) -> str | None:
if a.handles & b.handles or (a.variation and a.variation in b.handles) or (b.variation and b.variation in a.handles):
return "handle"
na, nb = _norm_name(a.name), _norm_name(b.name)
- if na and na == nb and len(na) > 3:
+ if na and na == nb and len(na) > 3 and na not in common_names:
return "name"
if _jaccard(_tokens(a.bio), _tokens(b.bio)) >= BIO_MIN_JACCARD:
return "bio"
return None
+def _common_names(profiles: list[Profile]) -> frozenset[str]:
+ """Normalized names shared by too many profiles to be identifying."""
+ from collections import Counter
+
+ counts: Counter[str] = Counter(
+ na for p in profiles if (na := _norm_name(p.name))
+ )
+ threshold = max(4, int(0.02 * len(profiles)))
+ return frozenset(name for name, count in counts.items() if count > threshold)
+
+
def cluster_profiles(profiles: list[Profile]) -> list[dict[str, Any]]:
"""Union-find clustering. Returns clusters as dicts (only multi-site clusters)."""
n = len(profiles)
uf = _UnionFind(n)
+ common_names = _common_names(profiles)
reasons: dict[tuple[int, int], str] = {}
for i in range(n):
for j in range(i + 1, n):
- reason = _pair_reason(profiles[i], profiles[j])
+ reason = _pair_reason(profiles[i], profiles[j], common_names)
if reason:
uf.union(i, j)
reasons[(i, j)] = reason
diff --git a/tests/test_new_features.py b/tests/test_new_features.py
index 86c9d6e..b31b99a 100644
--- a/tests/test_new_features.py
+++ b/tests/test_new_features.py
@@ -17,7 +17,10 @@ from aliens_eye.ml.collect import write_rows
from aliens_eye.selfcheck import _metrics
-def _profile_result(site, variation, status="Found", name="", bio="", avatar="", url=""):
+def _profile_result(
+ site, variation, status="Found", name="", bio="", avatar="", url="",
+ name_from_profile=True,
+):
return {
"site": site,
"url": url or f"https://{site}.com/{variation}",
@@ -28,7 +31,14 @@ def _profile_result(site, variation, status="Found", name="", bio="", avatar="",
"confidence": 90,
"ai_analysis": {
"features": {"http_200": 1.0},
- "signals": {"profile": {"name": name, "bio": bio, "avatar": avatar}},
+ "signals": {
+ "profile": {
+ "name": name,
+ "bio": bio,
+ "avatar": avatar,
+ "name_from_profile": name_from_profile,
+ }
+ },
},
}
@@ -115,7 +125,7 @@ def test_no_cluster_when_unrelated():
def test_profiles_from_report_skips_not_found():
rep = _report({
"alice": [
- _profile_result("github", "alice", status="Found", name="A"),
+ _profile_result("github", "alice", status="Found", name="Alice Smith"),
_profile_result("reddit", "alice", status="Not Found"),
]
})
@@ -123,6 +133,47 @@ def test_profiles_from_report_skips_not_found():
assert [p.site for p in profiles] == ["github"]
+def test_profiles_from_report_is_found_only():
+ # Maybe-band results (mostly bot-walls) must be excluded by default.
+ rep = _report({
+ "alice": [
+ _profile_result("github", "alice", status="Found", name="Alice Smith"),
+ _profile_result("reddit", "alice", status="Maybe", name="Alice Smith"),
+ ]
+ })
+ profiles = profiles_from_report(rep)
+ assert [p.site for p in profiles] == ["github"]
+
+
+def test_title_derived_name_not_used_for_identity():
+ # Same name on two sites, but it came from <title> (name_from_profile=False):
+ # must NOT cluster. Flip the flag on and it should.
+ junk = _report({
+ "alice": [
+ _profile_result("s1", "alice", name="Just a moment...", name_from_profile=False),
+ _profile_result("s2", "alice", name="Just a moment...", name_from_profile=False),
+ ]
+ })
+ assert cluster_profiles(profiles_from_report(junk)) == []
+
+ real = _report({
+ "alice": [
+ _profile_result("s1", "alice", name="Alice Smith", name_from_profile=True),
+ _profile_result("s2", "alice", name="Alice Smith", name_from_profile=True),
+ ]
+ })
+ assert len(cluster_profiles(profiles_from_report(real))) == 1
+
+
+def test_common_name_downweighted():
+ # A structured name shared by many profiles is boilerplate -> no cluster.
+ profiles = [
+ Profile("u", f"s{i}", name="Sign In", bio="", avatar="", url="", status="Found")
+ for i in range(10)
+ ]
+ assert cluster_profiles(profiles) == []
+
+
# --- expansion ------------------------------------------------------------
def test_candidate_usernames_from_bio():