🕵️♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.
Python cybersecuritycybersecurity-toolsemail-osintethical-hackingethical-hacking-toolsidentificationinfosecosintosint-emailosint-toolosint-toolsosint-usernamepentestingpythonredteam-toolsredteamingthreat-intelligence
fix(academia): detect profiles via embedded ld+json instead of a bare 200
Bruno Leonardo Michels854879dbb199
diff --git a/user_scanner/user_scan/community/academia.py b/user_scanner/user_scan/community/academia.pyindex 8637184..59d971e 100644--- a/user_scanner/user_scan/community/academia.py+++ b/user_scanner/user_scan/community/academia.py@@ -1,27 +1,159 @@-from user_scanner.core.helpers import get_random_user_agent-from user_scanner.core.orchestrator import Result, make_request+import html+import json+import re++from user_scanner.core.impersonate import impersonate_validate+from user_scanner.core.result import Result++# Academia.edu scopes handles per subdomain: the same slug can belong to a+# different person on an institution host (e.g. ucp.academia.edu). Only the+# independent-researcher namespace is checkable without enumerating every+# institution subdomain, so a verdict here covers that namespace alone.+BASE_URL = "https://independent.academia.edu"+def validate_academia(user: str) -> Result:+ url = f"{BASE_URL}/{user}"++ # Rails reads the segment after a dot as a format extension, so+ # "/John.Smith" serves the unrelated profile "/john". Such a handle is not+ # addressable and must not be turned into a verdict.if "." in user:- return Result.available("Username cannot contain periods")-- url = f"https://independent.academia.edu/{user}"-- headers = {- "User-Agent": get_random_user_agent(),- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",- "Accept-Language": "en-US,en;q=0.9",+ return Result.error("Handles containing a period are not addressable", url=url)++ def process(response):+ if response.status_code == 404:+ if "wrong aisle" in response.text:+ return Result.available(url=url)+ return Result.error("Unexpected 404 (not the not-found page)", url=url)++ if response.status_code != 200:+ return Result.error(f"Unexpected status: {response.status_code}", url=url)++ profile = _profile_json(response.text)+ if profile is None:+ return Result.error("Profile confirmation not found", url=url)++ # A miss redirects rather than 404s in some casings, and the dot route+ # above lands on a foreign profile, so the embedded canonical handle is+ # what confirms the page belongs to the requested user.+ handle = str(profile.get("mainEntity", {}).get("url", "")).rstrip("/").rsplit("/", 1)[-1]+ if handle.lower() != user.lower():+ return Result.error(f"Profile resolved to a different handle: {handle}", url=url)++ extra, media = _extract_profile(response.text, profile)+ return Result.taken(extra=extra, media=media, url=url)++ return impersonate_validate(url, process, show_url=url, allow_redirects=True)+++def _profile_json(html_text: str) -> dict | None:+ for block in re.findall(+ r'<script type="application/ld\+json">(.*?)</script>', html_text, re.DOTALL+ ):+ try:+ data = json.loads(block)+ except ValueError:+ continue+ if isinstance(data, dict) and data.get("@type") == "ProfilePage":+ return data+ return None+++def _extract_profile(html_text: str, profile: dict) -> tuple[dict, dict]:+ person = profile.get("mainEntity", {})+ extra: dict = {"name": person.get("name")}+ media: dict = {}++ avatar = person.get("image")+ if avatar and "no_pic" not in avatar:+ media["avatar"] = avatar++ for key, field in (("joined", "dateCreated"), ("updated", "dateModified")):+ value = profile.get(field)+ if value:+ extra[key] = value[:10]++ extra["bio"] = profile.get("description")++ links = person.get("sameAs") or profile.get("sameAs")+ if links:+ extra["links"] = ", ".join(links)++ affiliation = re.search(+ r'class="affiliations-container[^"]*"[^>]*>(.*?)</div>', html_text, re.DOTALL+ )+ if affiliation:+ extra["affiliation"] = _text(affiliation.group(1))++ # Every follow control on the page belongs to the profile owner; the+ # related-author cards carry no user id of their own.+ user_id = re.search(r'data-follow-user-id="(\d+)"', html_text)+ if user_id:+ extra["user_id"] = user_id.group(1)+ extra.update(_extract_interests(html_text, user_id.group(1)))++ extra.update(_extract_stats(html_text))+ extra.update(_extract_bio_details(html_text))++ return extra, media+++def _extract_interests(html_text: str, user_id: str) -> dict:+ interests = {}++ total = re.search(+ rf'data-has-card-for-ri-list="{user_id}"[^>]*>View All \((\d+)\)', html_text+ )+ if total:+ interests["interests_count"] = total.group(1)++ # The interest pills render as sibling React payloads; keying the enclosing+ # anchor on the owner's id keeps a recommendation carousel out of the list.+ labels: list[str] = []+ for anchor in re.findall(+ rf'data-has-card-for-ri-list="{user_id}"(.*?)</a>', html_text, re.DOTALL+ ):+ pill = re.search(r'data-component-name="Pill"[^>]*>(\{.*?\})</script>', anchor, re.DOTALL)+ if not pill:+ continue+ try:+ children = json.loads(pill.group(1)).get("children") or []+ except ValueError:+ continue+ labels.extend(str(child) for child in children)++ if labels:+ interests["interests"] = ", ".join(labels)++ return interests+++def _extract_stats(html_text: str) -> dict:+ region = re.search(r'class="user-stats-container"(.*?)<aside', html_text, re.DOTALL)+ if not region:+ return {}++ stats = {}+ for label, value in re.findall(+ r'<p class="label">(.*?)</p>\s*<p class="data">(.*?)</p>', region.group(1), re.DOTALL+ ):+ stats[_text(label)] = _text(value)+ return stats+++def _extract_bio_details(html_text: str) -> dict:+ bio = re.search(r'class="profile-bio[^"]*"[^>]*>(.*?)</div>', html_text, re.DOTALL)+ if not bio:+ return {}++ return {+ _text(label): _text(value)+ for label, value in re.findall(+ r'<span class="u-fw700">(.*?):.*?</span>(.*?)<br\s*/?>', bio.group(1), re.DOTALL+ )}-- try:- response = make_request(url, headers=headers, follow_redirects=True)-- if response.status_code == 200:- return Result.taken(url=url)- elif response.status_code == 404:- return Result.available(url=url)-- return Result.error(f"Unexpected response status: {response.status_code}", url=url)-- except Exception as e:- return Result.error(e, url=url)+++def _text(html_fragment: str) -> str:+ return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", "", html_fragment))).strip()