π΅οΈββοΈ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.
Python cybersecuritycybersecurity-toolsemail-osintethical-hackingethical-hacking-toolsidentificationinfosecosintosint-emailosint-toolosint-toolsosint-usernamepentestingpythonredteam-toolsredteamingthreat-intelligence
feat(gravatar): extract links, interests and profile details (#516)
BrunoLMcc228e7dd1f9
diff --git a/user_scanner/email_scan/social/gravatar.py b/user_scanner/email_scan/social/gravatar.pyindex 906126a..24533cf 100644--- a/user_scanner/email_scan/social/gravatar.py+++ b/user_scanner/email_scan/social/gravatar.py@@ -1,7 +1,95 @@import hashlib+import html+import re+from collections.abc import Iterator+import httpx++from user_scanner.core.helpers import get_global_timeoutfrom user_scanner.core.result import Result+PROFILE_HOST = "https://gravatar.com"+DEFAULT_TIMEOUT = 15.0++HEADERS = {+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"+}++# The public JSON APIs omit the profile's "Links" card, its interests and the+# advanced-details panel; only the rendered profile page carries them.+INTERESTS_RE = re.compile(r"<h2[^>]*>\s*Interests\s*</h2>.*?<ul[^>]*>(.*?)</ul>", re.S)+INTEREST_ITEM_RE = re.compile(r"<span>\s*([^<]+?)\s*</span>")+TITLE_RE = re.compile(r'title="([^"]*)"')+ADV_DETAIL_RE = re.compile(+ r"<span>\s*(Profile|Updated|Created|Languages|Time):\s*([^<]+?)\s*</span>"+)+# Both JSON APIs list only the providers their schema knows about β a YouTube+# connection is absent from each yet rendered on the page, so the card is the+# only complete source. Each entry is anchored on its checkmark's help link and+# the rel="me" of the account anchor rather than on styling hooks.+VERIFIED_MARKER = "support.gravatar.com/profiles/verified-accounts"+VERIFIED_LABEL_RE = re.compile(r">\s*([^<>]{1,60}?)\s*</span>\s*<a[^>]*$", re.S)+ANCHOR_RE = re.compile(r"<a\b[^>]*>", re.S)+REL_ME_RE = re.compile(r'rel="[^"]*\bme\b[^"]*"')+HREF_RE = re.compile(r'href="([^"]+)"')+TIMEZONE_RE = re.compile(r"\(([^)]+)\)")++ADV_DETAIL_KEYS = {+ "Profile": "profile_type",+ "Updated": "last_updated",+ "Created": "created",+ "Languages": "languages",+}+++async def validate_gravatar(email: str) -> Result:+ email_clean = email.lower().strip()+ timeout = get_global_timeout() or DEFAULT_TIMEOUT+ # Older profiles are only addressable by the legacy MD5 hash.+ hashes = [+ hashlib.sha256(email_clean.encode("utf-8")).hexdigest(),+ hashlib.md5(email_clean.encode("utf-8")).hexdigest(),+ ]+ try:+ async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:+ for email_hash in hashes:+ response = await client.get(+ f"https://www.gravatar.com/avatar/{email_hash}?d=404", headers=HEADERS+ )+ if response.status_code == 200:+ extra, media = await _collect_profile(client, email_hash)+ return Result.taken(+ url=extra.get("profile_url", PROFILE_HOST), extra=extra, media=media+ )+ if response.status_code != 404:+ return Result.error(f"HTTP {response.status_code}", url=PROFILE_HOST)+ return Result.available(url=PROFILE_HOST)+ except httpx.TimeoutException:+ return Result.error("Connection timed out", url=PROFILE_HOST)+ except Exception as e:+ return Result.error(e, url=PROFILE_HOST)+++async def _collect_profile(client: httpx.AsyncClient, email_hash: str) -> tuple[dict, dict]:+ extra: dict = {}+ media: dict = {"avatar": f"https://www.gravatar.com/avatar/{email_hash}"}+ try:+ response = await client.get(f"https://en.gravatar.com/{email_hash}.json", headers=HEADERS)+ if response.status_code == 200:+ entries = response.json().get("entry") or []+ if entries and isinstance(entries[0], dict):+ _extract_profile_data(entries[0], extra, media)+ except Exception:+ pass+ try:+ response = await client.get(f"{PROFILE_HOST}/{email_hash}", headers=HEADERS)+ if response.status_code == 200:+ _extract_page_data(response.text, extra)+ except Exception:+ pass+ return extra, media++def _extract_profile_data(entry: dict, extra: dict, media: dict) -> None:if entry.get("preferredUsername"):extra["username"] = str(entry["preferredUsername"]).strip()@@ -15,10 +103,16 @@ def _extract_profile_data(entry: dict, extra: dict, media: dict) -> None:extra["bio"] = str(entry["aboutMe"]).strip()if entry.get("currentLocation"):extra["location"] = str(entry["currentLocation"]).strip()- if entry.get("jobTitle"):- extra["job_title"] = str(entry["jobTitle"]).strip()+ # The legacy endpoint returns this as job_title; jobTitle is the v3 spelling.+ job_title = entry.get("jobTitle") or entry.get("job_title")+ if job_title:+ extra["job_title"] = str(job_title).strip()if entry.get("company"):extra["company"] = str(entry["company"]).strip()+ if entry.get("pronouns"):+ extra["pronouns"] = str(entry["pronouns"]).strip()+ if entry.get("pronunciation"):+ extra["pronunciation"] = str(entry["pronunciation"]).strip()name_info = entry.get("name")if isinstance(name_info, dict) and name_info.get("formatted"):@@ -41,10 +135,16 @@ def _extract_profile_data(entry: dict, extra: dict, media: dict) -> None:if not isinstance(acc, dict):continueacc_name = str(acc.get("name") or acc.get("shortname") or "Account")- acc_url = acc.get("url") or acc.get("display") or acc.get("username")- if acc_url is not None and str(acc_url).strip():- status = " (verified)" if acc.get("verified") else ""- acc_list.append(f"{acc_name}: {str(acc_url).strip()}{status}")+ acc_url = str(acc.get("url") or acc.get("display") or acc.get("username") or "").strip()+ status = " (verified)" if acc.get("verified") else ""+ # Providers that forbid public profile links (Facebook) are stored+ # pointing at Gravatar's help page, which says nothing about the user.+ if VERIFIED_MARKER in acc_url:+ acc_url = ""+ if acc_url:+ acc_list.append(f"{acc_name}: {acc_url}{status}")+ elif acc.get("verified"):+ acc_list.append(f"{acc_name}{status}")if acc_list:extra["verified_accounts"] = ", ".join(acc_list)@@ -58,6 +158,26 @@ def _extract_profile_data(entry: dict, extra: dict, media: dict) -> None:if url_list:extra["websites"] = ", ".join(url_list)+ contact_info = entry.get("contactInfo")+ if isinstance(contact_info, list):+ contact_list = [+ f"{str(c.get('type', 'contact'))}: {str(c['value']).strip()}"+ for c in contact_info+ if isinstance(c, dict) and c.get("value") is not None and str(c["value"]).strip()+ ]+ if contact_list:+ extra["contact_info"] = ", ".join(contact_list)++ phones = entry.get("phoneNumbers")+ if isinstance(phones, list):+ phone_list = [+ f"{str(p.get('type', 'phone'))}: {str(p['value']).strip()}"+ for p in phones+ if isinstance(p, dict) and p.get("value") is not None and str(p["value"]).strip()+ ]+ if phone_list:+ extra["phone_numbers"] = ", ".join(phone_list)+emails = entry.get("emails")if isinstance(emails, list):email_list = [@@ -78,62 +198,98 @@ def _extract_profile_data(entry: dict, extra: dict, media: dict) -> None:if crypto_list:extra["crypto_addresses"] = ", ".join(crypto_list)-async def _check(email: str) -> Result:- show_url = "https://gravatar.com"- email_clean = email.lower().strip()- email_hash = hashlib.sha256(email_clean.encode("utf-8")).hexdigest()- url = f"https://www.gravatar.com/avatar/{email_hash}?d=404"- headers = {- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"- }- try:- async with httpx.AsyncClient(timeout=15.0) as client:- response = await client.get(url, headers=headers)- if response.status_code == 200:- extra: dict = {}- media = {"avatar": f"https://www.gravatar.com/avatar/{email_hash}"}- profile_url = f"https://en.gravatar.com/{email_hash}.json"- try:- profile_resp = await client.get(profile_url, headers=headers, timeout=15.0)- if profile_resp.status_code == 200:- data = profile_resp.json()- entries = data.get("entry", [])- if entries and isinstance(entries, list):- _extract_profile_data(entries[0], extra, media)- except Exception:- pass- final_url = extra.get("profile_url", show_url)- return Result.taken(url=final_url, extra=extra, media=media)- elif response.status_code == 404:- # Also fall back to check MD5 since some older profiles might only map via MD5- email_md5 = hashlib.md5(email_clean.encode("utf-8")).hexdigest()- url_md5 = f"https://www.gravatar.com/avatar/{email_md5}?d=404"-- response_md5 = await client.get(url_md5, headers=headers)- if response_md5.status_code == 200:- extra = {}- media = {"avatar": f"https://www.gravatar.com/avatar/{email_md5}"}- profile_url_md5 = f"https://en.gravatar.com/{email_md5}.json"- try:- profile_resp = await client.get(profile_url_md5, headers=headers, timeout=15.0)- if profile_resp.status_code == 200:- data = profile_resp.json()- entries = data.get("entry", [])- if entries and isinstance(entries, list):- _extract_profile_data(entries[0], extra, media)- except Exception:- pass- final_url = extra.get("profile_url", show_url)- return Result.taken(url=final_url, extra=extra, media=media)- elif response_md5.status_code == 404:- return Result.available(url=show_url)- else:- return Result.error(f"HTTP MD5 {response_md5.status_code}", url=show_url)- return Result.error(f"HTTP {response.status_code}", url=show_url)- except httpx.TimeoutException:- return Result.error("Connection timed out", url=show_url)- except Exception as e:- return Result.error(e, url=show_url)+ background = entry.get("profileBackground")+ if isinstance(background, dict):+ if background.get("url"):+ media["background"] = str(background["url"]).strip()+ if background.get("color"):+ extra["background_color"] = str(background["color"]).strip()-async def validate_gravatar(email: str) -> Result:- return await _check(email)++def _extract_page_data(page: str, extra: dict) -> None:+ _merge_verified_accounts(page, extra)++ links = []+ for title, url in _iter_links(page):+ entry = f"{title}: {url}"+ if entry not in links:+ links.append(entry)+ if links:+ extra["links"] = ", ".join(links)++ interests = INTERESTS_RE.search(page)+ if interests:+ items = [html.unescape(i).strip() for i in INTEREST_ITEM_RE.findall(interests.group(1))]+ items = [i for i in items if i]+ if items:+ extra["interests"] = ", ".join(items)++ for label, value in ADV_DETAIL_RE.findall(page):+ value = html.unescape(value).strip()+ if not value or value == "-":+ continue+ if label == "Time":+ zone = TIMEZONE_RE.search(value)+ # Profiles with no timezone set render as UTC, so a bare "UTC" says+ # nothing about the owner and is dropped.+ if zone and zone.group(1).strip() != "UTC":+ extra["timezone"] = zone.group(1).strip()+ continue+ extra[ADV_DETAIL_KEYS[label]] = value+++def _merge_verified_accounts(page: str, extra: dict) -> None:+ existing = [e for e in str(extra.get("verified_accounts", "")).split(", ") if e]+ known_urls = {_normalize_url(e.rsplit(": ", 1)[-1]) for e in existing}+ # The same account can carry different URLs in each source β the JSON keeps+ # whatever the owner entered, the page renders a canonical one β so the+ # service name is what identifies a duplicate.+ known_services = {e.split(":", 1)[0].removesuffix(" (verified)").strip().lower() for e in existing}+ for label, url in _iter_verified_accounts(page):+ if _normalize_url(url) in known_urls or label.lower() in known_services:+ continue+ known_urls.add(_normalize_url(url))+ known_services.add(label.lower())+ existing.append(f"{label or 'Account'}: {url} (verified)")+ if existing:+ extra["verified_accounts"] = ", ".join(existing)+++def _iter_verified_accounts(page: str) -> Iterator[tuple[str, str]]:+ markers = [m.start() for m in re.finditer(re.escape(VERIFIED_MARKER), page)]+ for index, start in enumerate(markers):+ # The account anchor sits between this checkmark and the next entry's.+ end = markers[index + 1] if index + 1 < len(markers) else len(page)+ url = ""+ for tag in ANCHOR_RE.finditer(page, start, end):+ href = HREF_RE.search(tag.group(0))+ if href and REL_ME_RE.search(tag.group(0)):+ url = html.unescape(href.group(1)).strip()+ break+ # A help-page URL is the placeholder for providers that forbid public+ # links, and it also matches the marker, so it seeds phantom entries.+ if not url or VERIFIED_MARKER in url:+ continue+ label = VERIFIED_LABEL_RE.search(page, max(0, start - 300), start)+ yield (html.unescape(label.group(1)).strip() if label else ""), url+++def _iter_links(page: str) -> Iterator[tuple[str, str]]:+ # A personal link carries the owner's own caption; every other rel="me"+ # anchor on the page β the icon row and the verified cards β repeats its own+ # URL as the title.+ for tag in ANCHOR_RE.finditer(page):+ if not REL_ME_RE.search(tag.group(0)):+ continue+ href = HREF_RE.search(tag.group(0))+ title = TITLE_RE.search(tag.group(0))+ if not href or not title:+ continue+ url = html.unescape(href.group(1)).strip()+ caption = html.unescape(title.group(1)).strip()+ if url and caption and caption != url:+ yield caption, url+++def _normalize_url(value: str) -> str:+ return value.removesuffix(" (verified)").strip().rstrip("/").lower()