Skip to content

🕵️‍♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.

fix(academia): detect profiles via embedded ld+json instead of a bare 200

Bruno Leonardo Michels854879dbb199
diff --git a/user_scanner/user_scan/community/academia.py b/user_scanner/user_scan/community/academia.py
index 8637184..59d971e 100644
--- a/user_scanner/user_scan/community/academia.py
+++ b/user_scanner/user_scan/community/academia.py
@@ -1,27 +1,159 @@
-from user_scanner.core.helpers import get_random_user_agent
-from user_scanner.core.orchestrator import Result, make_request
+import html
+import json
+import re
+
+from user_scanner.core.impersonate import impersonate_validate
+from user_scanner.core.result import Result
+
+# Academia.edu scopes handles per subdomain: the same slug can belong to a
+# different person on an institution host (e.g. ucp.academia.edu). Only the
+# independent-researcher namespace is checkable without enumerating every
+# institution subdomain, so a verdict here covers that namespace alone.
+BASE_URL = "https://independent.academia.edu"
+
def validate_academia(user: str) -> Result:
+ url = f"{BASE_URL}/{user}"
+
+ # Rails reads the segment after a dot as a format extension, so
+ # "/John.Smith" serves the unrelated profile "/john". Such a handle is not
+ # addressable and must not be turned into a verdict.
if "." in user:
- return Result.available("Username cannot contain periods")
-
- url = f"https://independent.academia.edu/{user}"
-
- headers = {
- "User-Agent": get_random_user_agent(),
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
- "Accept-Language": "en-US,en;q=0.9",
+ return Result.error("Handles containing a period are not addressable", url=url)
+
+ def process(response):
+ if response.status_code == 404:
+ if "wrong aisle" in response.text:
+ return Result.available(url=url)
+ return Result.error("Unexpected 404 (not the not-found page)", url=url)
+
+ if response.status_code != 200:
+ return Result.error(f"Unexpected status: {response.status_code}", url=url)
+
+ profile = _profile_json(response.text)
+ if profile is None:
+ return Result.error("Profile confirmation not found", url=url)
+
+ # A miss redirects rather than 404s in some casings, and the dot route
+ # above lands on a foreign profile, so the embedded canonical handle is
+ # what confirms the page belongs to the requested user.
+ handle = str(profile.get("mainEntity", {}).get("url", "")).rstrip("/").rsplit("/", 1)[-1]
+ if handle.lower() != user.lower():
+ return Result.error(f"Profile resolved to a different handle: {handle}", url=url)
+
+ extra, media = _extract_profile(response.text, profile)
+ return Result.taken(extra=extra, media=media, url=url)
+
+ return impersonate_validate(url, process, show_url=url, allow_redirects=True)
+
+
+def _profile_json(html_text: str) -> dict | None:
+ for block in re.findall(
+ r'<script type="application/ld\+json">(.*?)</script>', html_text, re.DOTALL
+ ):
+ try:
+ data = json.loads(block)
+ except ValueError:
+ continue
+ if isinstance(data, dict) and data.get("@type") == "ProfilePage":
+ return data
+ return None
+
+
+def _extract_profile(html_text: str, profile: dict) -> tuple[dict, dict]:
+ person = profile.get("mainEntity", {})
+ extra: dict = {"name": person.get("name")}
+ media: dict = {}
+
+ avatar = person.get("image")
+ if avatar and "no_pic" not in avatar:
+ media["avatar"] = avatar
+
+ for key, field in (("joined", "dateCreated"), ("updated", "dateModified")):
+ value = profile.get(field)
+ if value:
+ extra[key] = value[:10]
+
+ extra["bio"] = profile.get("description")
+
+ links = person.get("sameAs") or profile.get("sameAs")
+ if links:
+ extra["links"] = ", ".join(links)
+
+ affiliation = re.search(
+ r'class="affiliations-container[^"]*"[^>]*>(.*?)</div>', html_text, re.DOTALL
+ )
+ if affiliation:
+ extra["affiliation"] = _text(affiliation.group(1))
+
+ # Every follow control on the page belongs to the profile owner; the
+ # related-author cards carry no user id of their own.
+ user_id = re.search(r'data-follow-user-id="(\d+)"', html_text)
+ if user_id:
+ extra["user_id"] = user_id.group(1)
+ extra.update(_extract_interests(html_text, user_id.group(1)))
+
+ extra.update(_extract_stats(html_text))
+ extra.update(_extract_bio_details(html_text))
+
+ return extra, media
+
+
+def _extract_interests(html_text: str, user_id: str) -> dict:
+ interests = {}
+
+ total = re.search(
+ rf'data-has-card-for-ri-list="{user_id}"[^>]*>View All \((\d+)\)', html_text
+ )
+ if total:
+ interests["interests_count"] = total.group(1)
+
+ # The interest pills render as sibling React payloads; keying the enclosing
+ # anchor on the owner's id keeps a recommendation carousel out of the list.
+ labels: list[str] = []
+ for anchor in re.findall(
+ rf'data-has-card-for-ri-list="{user_id}"(.*?)</a>', html_text, re.DOTALL
+ ):
+ pill = re.search(r'data-component-name="Pill"[^>]*>(\{.*?\})</script>', anchor, re.DOTALL)
+ if not pill:
+ continue
+ try:
+ children = json.loads(pill.group(1)).get("children") or []
+ except ValueError:
+ continue
+ labels.extend(str(child) for child in children)
+
+ if labels:
+ interests["interests"] = ", ".join(labels)
+
+ return interests
+
+
+def _extract_stats(html_text: str) -> dict:
+ region = re.search(r'class="user-stats-container"(.*?)<aside', html_text, re.DOTALL)
+ if not region:
+ return {}
+
+ stats = {}
+ for label, value in re.findall(
+ r'<p class="label">(.*?)</p>\s*<p class="data">(.*?)</p>', region.group(1), re.DOTALL
+ ):
+ stats[_text(label)] = _text(value)
+ return stats
+
+
+def _extract_bio_details(html_text: str) -> dict:
+ bio = re.search(r'class="profile-bio[^"]*"[^>]*>(.*?)</div>', html_text, re.DOTALL)
+ if not bio:
+ return {}
+
+ return {
+ _text(label): _text(value)
+ for label, value in re.findall(
+ r'<span class="u-fw700">(.*?):.*?</span>(.*?)<br\s*/?>', bio.group(1), re.DOTALL
+ )
}
-
- try:
- response = make_request(url, headers=headers, follow_redirects=True)
-
- if response.status_code == 200:
- return Result.taken(url=url)
- elif response.status_code == 404:
- return Result.available(url=url)
-
- return Result.error(f"Unexpected response status: {response.status_code}", url=url)
-
- except Exception as e:
- return Result.error(e, url=url)
+
+
+def _text(html_fragment: str) -> str:
+ return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", "", html_fragment))).strip()