🕵️♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.
Python cybersecuritycybersecurity-toolsemail-osintethical-hackingethical-hacking-toolsidentificationinfosecosintosint-emailosint-toolosint-toolsosint-usernamepentestingpythonredteam-toolsredteamingthreat-intelligence
Merge pull request #663 from kristoisberg/t3code/add-luma-username-module
Kristo Isbergf11a5253dfec
diff --git a/tests/test_pivots.py b/tests/test_pivots.pyindex c297658..c53be16 100644--- a/tests/test_pivots.py+++ b/tests/test_pivots.py@@ -32,6 +32,10 @@ def make_result(site_name="Gravatar", extra=None, found=True, **kwargs):("https://github.com/johndoe", ("github", "johndoe")),("https://www.linkedin.com/in/johndoe/", ("linkedin", "johndoe")),("https://br.linkedin.com/in/johndoe", ("linkedin", "johndoe")),+ (+ "https://www.linkedin.com/company/acme-inc/",+ ("linkedin_company", "acme-inc"),+ ),("https://x.com/JohnDoe2", ("x", "JohnDoe2")),("https://twitter.com/JohnDoe2", ("x", "JohnDoe2")),("https://stackoverflow.com/users/12345/johndoe", ("stackoverflow", "johndoe")),@@ -141,6 +145,20 @@ def test_platform_keys_are_read_through_their_suffixes(key, expected_site):assert pivot.site == expected_site+ "value,expected",+ [+ ("/in/jane-doe", ("linkedin", "jane-doe")),+ ("/company/acme-inc", ("linkedin_company", "acme-inc")),+ ],+)+def test_linkedin_handle_keeps_its_namespace_in_the_path(value, expected):+ result = make_result(site_name="Luma", extra={"linkedin_handle": value})++ (pivot,) = extract_pivots([result])+ assert (pivot.site, pivot.username) == expected++def test_an_identifier_field_is_not_read_as_a_handle():"""``_id`` is not a handle suffix, so YouTube's channel id stays out."""result = make_result(diff --git a/user_scanner/core/pivots.py b/user_scanner/core/pivots.pyindex 419e07a..3fc9fec 100644--- a/user_scanner/core/pivots.py+++ b/user_scanner/core/pivots.py@@ -168,6 +168,11 @@ _AT = r"^/@(?P<user>[^/?#]+)/?$"_HOST_ROUTES: Tuple[Tuple[Tuple[str, ...], str, Tuple[str, ...]], ...] = ((("x.com", "twitter.com"), "x", (_BARE,)),(("linkedin.com",), "linkedin", (r"^/in/(?P<user>[^/?#]+)/?$",)),+ (+ ("linkedin.com",),+ "linkedin_company",+ (r"^/company/(?P<user>[^/?#]+)/?$",),+ ),(("github.com",), "github", (_BARE,)),(("gist.github.com",), "githubgist", (_BARE,)),(("gitlab.com",), "gitlab", (_BARE,)),@@ -275,11 +280,12 @@ _SUBDOMAIN_ROUTES: Tuple[Tuple[str, str], ...] = (("deviantart.com", "deviantart"),)-_ROUTES = {- host: (module, tuple(re.compile(p) for p in patterns))- for hosts, module, patterns in _HOST_ROUTES- for host in hosts-}+_ROUTES: dict[str, list[tuple[str, tuple[re.Pattern, ...]]]] = {}+for hosts, module, patterns in _HOST_ROUTES:+ for host in hosts:+ _ROUTES.setdefault(host, []).append(+ (module, tuple(re.compile(pattern) for pattern in patterns))+ )class PivotKind(Enum):@@ -461,16 +467,16 @@ def resolve_url(url: str) -> Tuple[Optional[str], Optional[str]]:return subdomainfor candidate in _host_candidates(host):- route = _ROUTES.get(candidate)- if not route:+ routes = _ROUTES.get(candidate)+ if not routes:continue- module, patterns = route- for pattern in patterns:- match = pattern.match(path)- if match:- user = _clean_handle(match.group("user"), module)- if user:- return module, user+ for module, patterns in routes:+ for pattern in patterns:+ match = pattern.match(path)+ if match:+ user = _clean_handle(match.group("user"), module)+ if user:+ return module, user# A routed host whose path fits no profile shape is one of that site's# own pages, so the domain fallback must not fire for it.return None, None@@ -508,6 +514,13 @@ def _pivots_from_result(result: Result) -> Iterator[Pivot]:module = _platform_key_module(key)if module and not _URL_RE.search(value):+ if module == "linkedin" and value.startswith(("/in/", "/company/")):+ linked_module, handle = resolve_url(f"https://linkedin.com{value}")+ if handle:+ yield Pivot(+ handle, PivotKind.LINK, source_site, key, linked_module+ )+ continuehandle = _clean_handle(value, module)if handle:yield Pivot(handle, PivotKind.LINK, source_site, key, module)diff --git a/user_scanner/user_scan/creator/luma.py b/user_scanner/user_scan/creator/luma.pynew file mode 100644index 0000000..eac2341--- /dev/null+++ b/user_scanner/user_scan/creator/luma.py@@ -0,0 +1,53 @@+from urllib.parse import quote++from user_scanner.core.nextjs import parse_next_pages_data+from user_scanner.core.orchestrator import generic_validate+from user_scanner.core.result import Result+++def validate_luma(user: str) -> Result:+ url = f"https://luma.com/user/{quote(user, safe='')}"++ def process(response):+ if response.status_code != 200:+ return Result.error(f"Unexpected response status: {response.status_code}")++ data = parse_next_pages_data(response.text)+ if not data or data.get("page") != "/user/[...username]":+ return Result.error("Luma profile data was missing")++ page_props = data.get("props", {}).get("pageProps", {})+ initial_data = page_props.get("initialData")+ if page_props.get("status") == 404 and initial_data is None:+ return Result.available()+ if not isinstance(initial_data, dict):+ return Result.error("Unexpected Luma profile data")++ profile = initial_data.get("user")+ username = profile.get("username") if isinstance(profile, dict) else None+ if not isinstance(username, str) or username.casefold() != user.casefold():+ return Result.error("Luma profile did not match the requested username")++ return Result.taken(+ extra={+ "uid": profile.get("api_id"),+ "display_name": profile.get("name"),+ "first_name": profile.get("first_name"),+ "last_name": profile.get("last_name"),+ "bio": profile.get("bio_short"),+ "verified": profile.get("is_verified"),+ "timezone": profile.get("timezone"),+ "joined_at": initial_data.get("joined_at"),+ "events_hosted": initial_data.get("event_hosted_count"),+ "events_attended": initial_data.get("event_attended_count"),+ "website": profile.get("website"),+ "instagram_handle": profile.get("instagram_handle"),+ "linkedin_handle": profile.get("linkedin_handle"),+ "tiktok_handle": profile.get("tiktok_handle"),+ "twitter_handle": profile.get("twitter_handle"),+ "youtube_handle": profile.get("youtube_handle"),+ },+ media={"avatar": profile.get("avatar_url")},+ )++ return generic_validate(url, process, show_url=url)diff --git a/user_scanner/user_scan/social/linkedin.py b/user_scanner/user_scan/social/linkedin.pyindex 32fd9d2..f99ee0e 100644--- a/user_scanner/user_scan/social/linkedin.py+++ b/user_scanner/user_scan/social/linkedin.py@@ -17,7 +17,6 @@ HEADERS = {"User-Agent": "Twitterbot/1.0"}def validate_linkedin(user: str) -> Result:url = f"https://www.linkedin.com/in/{user}"- show_url = f"{url}/"def inner(response: httpx.Response) -> Result:status = response.status_code@@ -34,12 +33,15 @@ def validate_linkedin(user: str) -> Result:return Result.error(f"[{status}] Status didn't match. Report this on Github.")return generic_validate(- url, inner, show_url=show_url, headers=HEADERS, follow_redirects=True+ url, inner, show_url=f"{url}/", headers=HEADERS, follow_redirects=True)def _extract(text: str) -> tuple[dict, dict]:- person = _find_person(text)+ nodes = list(_ld_nodes(text))+ person: dict = next(+ (node for node in nodes if node.get("@type") == "Person"), {}+ )extra: dict = {}media: dict = {}@@ -86,10 +88,10 @@ def _extract(text: str) -> tuple[dict, dict]:if awards := list(dict.fromkeys(awards)):extra["awards"] = "; ".join(awards)- if articles := _find_articles(text):+ if articles := _find_articles(nodes):extra["articles"] = "; ".join(articles)- if last_published := _latest_published(text):+ if last_published := _latest_published(nodes):extra["last_posted"] = last_publishedif similar := _similar_profiles(text):@@ -111,16 +113,9 @@ def _extract(text: str) -> tuple[dict, dict]:return extra, media-def _find_person(text: str) -> dict:- for node in _ld_nodes(text):- if node.get("@type") == "Person":- return node- return {}---def _find_articles(text: str) -> list[str]:+def _find_articles(nodes: list[dict]) -> list[str]:articles = []- for node in _ld_nodes(text):+ for node in nodes:if node.get("@type") != "Article" or not (headline := node.get("headline")):continuedate = (node.get("datePublished") or "")[:10]@@ -128,17 +123,15 @@ def _find_articles(text: str) -> list[str]:return list(dict.fromkeys(articles))-def _latest_published(text: str) -> str | None:+def _latest_published(nodes: list[dict]) -> str | None:# Newest publish date across long-form Articles and Activity-feed posts# (DiscussionForumPosting); the ISO-8601 prefix sorts chronologically.dates = [node["datePublished"]- for node in _ld_nodes(text)+ for node in nodesif node.get("@type") in ("Article", "DiscussionForumPosting") and node.get("datePublished")]- if not dates:- return None- return max(dates)[:10]+ return max(dates)[:10] if dates else Nonedef _similar_profiles(text: str) -> list[str]:@@ -209,6 +202,7 @@ def _clean_html(value: str | None) -> str:def _og(text: str, prop: str) -> str | None:- match = re.search(rf'<meta[^>]+property="{re.escape(prop)}"[^>]+content="([^"]*)"', text) or \- re.search(rf'<meta[^>]+name="{re.escape(prop)}"[^>]+content="([^"]*)"', text)+ match = re.search(+ rf'<meta[^>]+property="{re.escape(prop)}"[^>]+content="([^"]*)"', text+ )return html.unescape(match.group(1)) if match else Nonediff --git a/user_scanner/user_scan/social/linkedin_company.py b/user_scanner/user_scan/social/linkedin_company.pynew file mode 100644index 0000000..cd28539--- /dev/null+++ b/user_scanner/user_scan/social/linkedin_company.py@@ -0,0 +1,118 @@+import html+import json+import re+from urllib.parse import quote, urlsplit++import httpx++from user_scanner.core.orchestrator import generic_validate+from user_scanner.core.result import Result++HEADERS = {"User-Agent": "Twitterbot/1.0"}++ABOUT_KEYS = {+ "industry": "industry",+ "size": "company_size",+ "organizationType": "organization_type",+ "foundedOn": "founded",+ "specialties": "specialties",+}+ABOUT_RE = re.compile(+ rf'data-test-id="about-us__({"|".join(ABOUT_KEYS)})"[^>]*>.*?<dd[^>]*>(.*?)</dd>',+ re.DOTALL,+)+++def _ld_nodes(text: str):+ for block in re.findall(+ r'<script type="application/ld\+json">(.*?)</script>', text, re.DOTALL+ ):+ try:+ data = json.loads(block)+ except json.JSONDecodeError:+ continue+ yield from data.get("@graph", [data])+++def _clean_html(value: str | None) -> str:+ if not value:+ return ""+ value = re.sub(r"<br\s*/?>", " ", value, flags=re.IGNORECASE)+ value = re.sub(r"<[^>]+>", "", value)+ return re.sub(r"\s+", " ", html.unescape(value)).strip()+++def _og(text: str, prop: str) -> str | None:+ match = re.search(+ rf'<meta[^>]+property="{re.escape(prop)}"[^>]+content="([^"]*)"', text+ )+ return html.unescape(match.group(1)) if match else None+++def _latest_published(nodes: list[dict]) -> str | None:+ dates = [+ node["datePublished"]+ for node in nodes+ if node.get("@type") in ("Article", "DiscussionForumPosting")+ and node.get("datePublished")+ ]+ return max(dates)[:10] if dates else None+++def validate_linkedin_company(user: str) -> Result:+ url = f"https://www.linkedin.com/company/{quote(user, safe='')}"++ def process(response: httpx.Response) -> Result:+ text = response.text+ if response.status_code == 404 and "<title>LinkedIn</title>" in text:+ return Result.available()+ if response.status_code != 200:+ return Result.error(f"Unexpected response status: {response.status_code}")++ nodes = list(_ld_nodes(text))+ organization = next(+ (node for node in nodes if node.get("@type") == "Organization"),+ None,+ )+ canonical = organization.get("url") if organization else None+ canonical_path = urlsplit(canonical).path if canonical else ""+ if not organization or not canonical_path.startswith("/company/"):+ return Result.error("LinkedIn company profile markers were missing")++ address = organization.get("address") or {}+ employees = organization.get("numberOfEmployees") or {}+ followers = re.search(+ r"([\d,]+) followers on LinkedIn", _og(text, "og:description") or ""+ )+ logo = organization.get("logo") or {}+ canonical_handle = canonical_path.rstrip("/").rsplit("/", 1)[-1]+ about = {+ ABOUT_KEYS[key]: _clean_html(value)+ for key, value in ABOUT_RE.findall(text)+ }++ return Result.taken(+ extra={+ "name": organization.get("name"),+ "description": _clean_html(organization.get("description")),+ "slogan": organization.get("slogan"),+ "website": organization.get("sameAs"),+ "city": address.get("addressLocality"),+ "region": address.get("addressRegion"),+ "country": address.get("addressCountry"),+ "employees": employees.get("value"),+ "followers": followers.group(1) if followers else None,+ "last_posted": _latest_published(nodes),+ **about,+ "canonical_handle": (+ canonical_handle+ if canonical_handle.casefold() != user.casefold()+ else None+ ),+ },+ media={"logo": logo.get("contentUrl") or _og(text, "og:image")},+ )++ return generic_validate(+ url, process, show_url=f"{url}/", headers=HEADERS, follow_redirects=True+ )