Skip to content

🕵️‍♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.

Merge pull request #663 from kristoisberg/t3code/add-luma-username-module

Kristo Isbergf11a5253dfec
diff --git a/tests/test_pivots.py b/tests/test_pivots.py
index c297658..c53be16 100644
--- a/tests/test_pivots.py
+++ b/tests/test_pivots.py
@@ -32,6 +32,10 @@ def make_result(site_name="Gravatar", extra=None, found=True, **kwargs):
("https://github.com/johndoe", ("github", "johndoe")),
("https://www.linkedin.com/in/johndoe/", ("linkedin", "johndoe")),
("https://br.linkedin.com/in/johndoe", ("linkedin", "johndoe")),
+ (
+ "https://www.linkedin.com/company/acme-inc/",
+ ("linkedin_company", "acme-inc"),
+ ),
("https://x.com/JohnDoe2", ("x", "JohnDoe2")),
("https://twitter.com/JohnDoe2", ("x", "JohnDoe2")),
("https://stackoverflow.com/users/12345/johndoe", ("stackoverflow", "johndoe")),
@@ -141,6 +145,20 @@ def test_platform_keys_are_read_through_their_suffixes(key, expected_site):
assert pivot.site == expected_site
+ "value,expected",
+ [
+ ("/in/jane-doe", ("linkedin", "jane-doe")),
+ ("/company/acme-inc", ("linkedin_company", "acme-inc")),
+ ],
+)
+def test_linkedin_handle_keeps_its_namespace_in_the_path(value, expected):
+ result = make_result(site_name="Luma", extra={"linkedin_handle": value})
+
+ (pivot,) = extract_pivots([result])
+ assert (pivot.site, pivot.username) == expected
+
+
def test_an_identifier_field_is_not_read_as_a_handle():
"""``_id`` is not a handle suffix, so YouTube's channel id stays out."""
result = make_result(
diff --git a/user_scanner/core/pivots.py b/user_scanner/core/pivots.py
index 419e07a..3fc9fec 100644
--- a/user_scanner/core/pivots.py
+++ b/user_scanner/core/pivots.py
@@ -168,6 +168,11 @@ _AT = r"^/@(?P<user>[^/?#]+)/?$"
_HOST_ROUTES: Tuple[Tuple[Tuple[str, ...], str, Tuple[str, ...]], ...] = (
(("x.com", "twitter.com"), "x", (_BARE,)),
(("linkedin.com",), "linkedin", (r"^/in/(?P<user>[^/?#]+)/?$",)),
+ (
+ ("linkedin.com",),
+ "linkedin_company",
+ (r"^/company/(?P<user>[^/?#]+)/?$",),
+ ),
(("github.com",), "github", (_BARE,)),
(("gist.github.com",), "githubgist", (_BARE,)),
(("gitlab.com",), "gitlab", (_BARE,)),
@@ -275,11 +280,12 @@ _SUBDOMAIN_ROUTES: Tuple[Tuple[str, str], ...] = (
("deviantart.com", "deviantart"),
)
-_ROUTES = {
- host: (module, tuple(re.compile(p) for p in patterns))
- for hosts, module, patterns in _HOST_ROUTES
- for host in hosts
-}
+_ROUTES: dict[str, list[tuple[str, tuple[re.Pattern, ...]]]] = {}
+for hosts, module, patterns in _HOST_ROUTES:
+ for host in hosts:
+ _ROUTES.setdefault(host, []).append(
+ (module, tuple(re.compile(pattern) for pattern in patterns))
+ )
class PivotKind(Enum):
@@ -461,16 +467,16 @@ def resolve_url(url: str) -> Tuple[Optional[str], Optional[str]]:
return subdomain
for candidate in _host_candidates(host):
- route = _ROUTES.get(candidate)
- if not route:
+ routes = _ROUTES.get(candidate)
+ if not routes:
continue
- module, patterns = route
- for pattern in patterns:
- match = pattern.match(path)
- if match:
- user = _clean_handle(match.group("user"), module)
- if user:
- return module, user
+ for module, patterns in routes:
+ for pattern in patterns:
+ match = pattern.match(path)
+ if match:
+ user = _clean_handle(match.group("user"), module)
+ if user:
+ return module, user
# A routed host whose path fits no profile shape is one of that site's
# own pages, so the domain fallback must not fire for it.
return None, None
@@ -508,6 +514,13 @@ def _pivots_from_result(result: Result) -> Iterator[Pivot]:
module = _platform_key_module(key)
if module and not _URL_RE.search(value):
+ if module == "linkedin" and value.startswith(("/in/", "/company/")):
+ linked_module, handle = resolve_url(f"https://linkedin.com{value}")
+ if handle:
+ yield Pivot(
+ handle, PivotKind.LINK, source_site, key, linked_module
+ )
+ continue
handle = _clean_handle(value, module)
if handle:
yield Pivot(handle, PivotKind.LINK, source_site, key, module)
diff --git a/user_scanner/user_scan/creator/luma.py b/user_scanner/user_scan/creator/luma.py
new file mode 100644
index 0000000..eac2341
--- /dev/null
+++ b/user_scanner/user_scan/creator/luma.py
@@ -0,0 +1,53 @@
+from urllib.parse import quote
+
+from user_scanner.core.nextjs import parse_next_pages_data
+from user_scanner.core.orchestrator import generic_validate
+from user_scanner.core.result import Result
+
+
+def validate_luma(user: str) -> Result:
+ url = f"https://luma.com/user/{quote(user, safe='')}"
+
+ def process(response):
+ if response.status_code != 200:
+ return Result.error(f"Unexpected response status: {response.status_code}")
+
+ data = parse_next_pages_data(response.text)
+ if not data or data.get("page") != "/user/[...username]":
+ return Result.error("Luma profile data was missing")
+
+ page_props = data.get("props", {}).get("pageProps", {})
+ initial_data = page_props.get("initialData")
+ if page_props.get("status") == 404 and initial_data is None:
+ return Result.available()
+ if not isinstance(initial_data, dict):
+ return Result.error("Unexpected Luma profile data")
+
+ profile = initial_data.get("user")
+ username = profile.get("username") if isinstance(profile, dict) else None
+ if not isinstance(username, str) or username.casefold() != user.casefold():
+ return Result.error("Luma profile did not match the requested username")
+
+ return Result.taken(
+ extra={
+ "uid": profile.get("api_id"),
+ "display_name": profile.get("name"),
+ "first_name": profile.get("first_name"),
+ "last_name": profile.get("last_name"),
+ "bio": profile.get("bio_short"),
+ "verified": profile.get("is_verified"),
+ "timezone": profile.get("timezone"),
+ "joined_at": initial_data.get("joined_at"),
+ "events_hosted": initial_data.get("event_hosted_count"),
+ "events_attended": initial_data.get("event_attended_count"),
+ "website": profile.get("website"),
+ "instagram_handle": profile.get("instagram_handle"),
+ "linkedin_handle": profile.get("linkedin_handle"),
+ "tiktok_handle": profile.get("tiktok_handle"),
+ "twitter_handle": profile.get("twitter_handle"),
+ "youtube_handle": profile.get("youtube_handle"),
+ },
+ media={"avatar": profile.get("avatar_url")},
+ )
+
+ return generic_validate(url, process, show_url=url)
diff --git a/user_scanner/user_scan/social/linkedin.py b/user_scanner/user_scan/social/linkedin.py
index 32fd9d2..f99ee0e 100644
--- a/user_scanner/user_scan/social/linkedin.py
+++ b/user_scanner/user_scan/social/linkedin.py
@@ -17,7 +17,6 @@ HEADERS = {"User-Agent": "Twitterbot/1.0"}
def validate_linkedin(user: str) -> Result:
url = f"https://www.linkedin.com/in/{user}"
- show_url = f"{url}/"
def inner(response: httpx.Response) -> Result:
status = response.status_code
@@ -34,12 +33,15 @@ def validate_linkedin(user: str) -> Result:
return Result.error(f"[{status}] Status didn't match. Report this on Github.")
return generic_validate(
- url, inner, show_url=show_url, headers=HEADERS, follow_redirects=True
+ url, inner, show_url=f"{url}/", headers=HEADERS, follow_redirects=True
)
def _extract(text: str) -> tuple[dict, dict]:
- person = _find_person(text)
+ nodes = list(_ld_nodes(text))
+ person: dict = next(
+ (node for node in nodes if node.get("@type") == "Person"), {}
+ )
extra: dict = {}
media: dict = {}
@@ -86,10 +88,10 @@ def _extract(text: str) -> tuple[dict, dict]:
if awards := list(dict.fromkeys(awards)):
extra["awards"] = "; ".join(awards)
- if articles := _find_articles(text):
+ if articles := _find_articles(nodes):
extra["articles"] = "; ".join(articles)
- if last_published := _latest_published(text):
+ if last_published := _latest_published(nodes):
extra["last_posted"] = last_published
if similar := _similar_profiles(text):
@@ -111,16 +113,9 @@ def _extract(text: str) -> tuple[dict, dict]:
return extra, media
-def _find_person(text: str) -> dict:
- for node in _ld_nodes(text):
- if node.get("@type") == "Person":
- return node
- return {}
-
-
-def _find_articles(text: str) -> list[str]:
+def _find_articles(nodes: list[dict]) -> list[str]:
articles = []
- for node in _ld_nodes(text):
+ for node in nodes:
if node.get("@type") != "Article" or not (headline := node.get("headline")):
continue
date = (node.get("datePublished") or "")[:10]
@@ -128,17 +123,15 @@ def _find_articles(text: str) -> list[str]:
return list(dict.fromkeys(articles))
-def _latest_published(text: str) -> str | None:
+def _latest_published(nodes: list[dict]) -> str | None:
# Newest publish date across long-form Articles and Activity-feed posts
# (DiscussionForumPosting); the ISO-8601 prefix sorts chronologically.
dates = [
node["datePublished"]
- for node in _ld_nodes(text)
+ for node in nodes
if node.get("@type") in ("Article", "DiscussionForumPosting") and node.get("datePublished")
]
- if not dates:
- return None
- return max(dates)[:10]
+ return max(dates)[:10] if dates else None
def _similar_profiles(text: str) -> list[str]:
@@ -209,6 +202,7 @@ def _clean_html(value: str | None) -> str:
def _og(text: str, prop: str) -> str | None:
- match = re.search(rf'<meta[^>]+property="{re.escape(prop)}"[^>]+content="([^"]*)"', text) or \
- re.search(rf'<meta[^>]+name="{re.escape(prop)}"[^>]+content="([^"]*)"', text)
+ match = re.search(
+ rf'<meta[^>]+property="{re.escape(prop)}"[^>]+content="([^"]*)"', text
+ )
return html.unescape(match.group(1)) if match else None
diff --git a/user_scanner/user_scan/social/linkedin_company.py b/user_scanner/user_scan/social/linkedin_company.py
new file mode 100644
index 0000000..cd28539
--- /dev/null
+++ b/user_scanner/user_scan/social/linkedin_company.py
@@ -0,0 +1,118 @@
+import html
+import json
+import re
+from urllib.parse import quote, urlsplit
+
+import httpx
+
+from user_scanner.core.orchestrator import generic_validate
+from user_scanner.core.result import Result
+
+HEADERS = {"User-Agent": "Twitterbot/1.0"}
+
+ABOUT_KEYS = {
+ "industry": "industry",
+ "size": "company_size",
+ "organizationType": "organization_type",
+ "foundedOn": "founded",
+ "specialties": "specialties",
+}
+ABOUT_RE = re.compile(
+ rf'data-test-id="about-us__({"|".join(ABOUT_KEYS)})"[^>]*>.*?<dd[^>]*>(.*?)</dd>',
+ re.DOTALL,
+)
+
+
+def _ld_nodes(text: str):
+ for block in re.findall(
+ r'<script type="application/ld\+json">(.*?)</script>', text, re.DOTALL
+ ):
+ try:
+ data = json.loads(block)
+ except json.JSONDecodeError:
+ continue
+ yield from data.get("@graph", [data])
+
+
+def _clean_html(value: str | None) -> str:
+ if not value:
+ return ""
+ value = re.sub(r"<br\s*/?>", " ", value, flags=re.IGNORECASE)
+ value = re.sub(r"<[^>]+>", "", value)
+ return re.sub(r"\s+", " ", html.unescape(value)).strip()
+
+
+def _og(text: str, prop: str) -> str | None:
+ match = re.search(
+ rf'<meta[^>]+property="{re.escape(prop)}"[^>]+content="([^"]*)"', text
+ )
+ return html.unescape(match.group(1)) if match else None
+
+
+def _latest_published(nodes: list[dict]) -> str | None:
+ dates = [
+ node["datePublished"]
+ for node in nodes
+ if node.get("@type") in ("Article", "DiscussionForumPosting")
+ and node.get("datePublished")
+ ]
+ return max(dates)[:10] if dates else None
+
+
+def validate_linkedin_company(user: str) -> Result:
+ url = f"https://www.linkedin.com/company/{quote(user, safe='')}"
+
+ def process(response: httpx.Response) -> Result:
+ text = response.text
+ if response.status_code == 404 and "<title>LinkedIn</title>" in text:
+ return Result.available()
+ if response.status_code != 200:
+ return Result.error(f"Unexpected response status: {response.status_code}")
+
+ nodes = list(_ld_nodes(text))
+ organization = next(
+ (node for node in nodes if node.get("@type") == "Organization"),
+ None,
+ )
+ canonical = organization.get("url") if organization else None
+ canonical_path = urlsplit(canonical).path if canonical else ""
+ if not organization or not canonical_path.startswith("/company/"):
+ return Result.error("LinkedIn company profile markers were missing")
+
+ address = organization.get("address") or {}
+ employees = organization.get("numberOfEmployees") or {}
+ followers = re.search(
+ r"([\d,]+) followers on LinkedIn", _og(text, "og:description") or ""
+ )
+ logo = organization.get("logo") or {}
+ canonical_handle = canonical_path.rstrip("/").rsplit("/", 1)[-1]
+ about = {
+ ABOUT_KEYS[key]: _clean_html(value)
+ for key, value in ABOUT_RE.findall(text)
+ }
+
+ return Result.taken(
+ extra={
+ "name": organization.get("name"),
+ "description": _clean_html(organization.get("description")),
+ "slogan": organization.get("slogan"),
+ "website": organization.get("sameAs"),
+ "city": address.get("addressLocality"),
+ "region": address.get("addressRegion"),
+ "country": address.get("addressCountry"),
+ "employees": employees.get("value"),
+ "followers": followers.group(1) if followers else None,
+ "last_posted": _latest_published(nodes),
+ **about,
+ "canonical_handle": (
+ canonical_handle
+ if canonical_handle.casefold() != user.casefold()
+ else None
+ ),
+ },
+ media={"logo": logo.get("contentUrl") or _og(text, "og:image")},
+ )
+
+ return generic_validate(
+ url, process, show_url=f"{url}/", headers=HEADERS, follow_redirects=True
+ )