🕵️♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.
Python cybersecuritycybersecurity-toolsemail-osintethical-hackingethical-hacking-toolsidentificationinfosecosintosint-emailosint-toolosint-toolsosint-usernamepentestingpythonredteam-toolsredteamingthreat-intelligence
feat: Next.js helper functions (#583)
Kristo Isbergd1f5bbaad374
diff --git a/tests/test_nextjs.py b/tests/test_nextjs.pynew file mode 100644index 0000000..7fafc3b--- /dev/null+++ b/tests/test_nextjs.py@@ -0,0 +1,41 @@+import json++from user_scanner.core.nextjs import (+ iter_next_app_flight_chunks,+ parse_next_pages_data,+ parse_next_pages_redirect,+)+++def test_parse_next_pages_data():+ document = """+ <script nonce="abc" id="__NEXT_DATA__" type="application/json">+ {"props":{"pageProps":{"username":"alice"}}}+ </script>+ """+ assert parse_next_pages_data(document) == {+ "props": {"pageProps": {"username": "alice"}}+ }+ assert parse_next_pages_data('<script id="__NEXT_DATA__">{bad}</script>') is None+++def test_parse_next_pages_data_unescapes_encoded_json():+ document = (+ '<script id="__NEXT_DATA__">'+ "{"props":{"pageProps":{}}}"+ "</script>"+ )+ assert parse_next_pages_data(document) == {"props": {"pageProps": {}}}+++def test_next_app_flight_chunks_and_pages_redirect():+ chunks = ['0:{"followersCount":7}', '1:{"name":"Alice"}']+ document = "".join(+ f"<script>self.__next_f.push([1,{json.dumps(chunk)}])</script>"+ for chunk in chunks+ )+ assert list(iter_next_app_flight_chunks(document)) == chunks+ assert parse_next_pages_redirect(+ {"__N_REDIRECT": "/", "__N_REDIRECT_STATUS": 307}+ ) == ("/", 307)+ assert parse_next_pages_redirect({"__N_REDIRECT": "/"}) is Nonediff --git a/user_scanner/core/nextjs.py b/user_scanner/core/nextjs.pynew file mode 100644index 0000000..7b4f915--- /dev/null+++ b/user_scanner/core/nextjs.py@@ -0,0 +1,49 @@+import html+import json+import re+from collections.abc import Iterator++_NEXT_PAGES_DATA_RE = re.compile(+ r'<script\b[^>]*\bid=["\']__NEXT_DATA__["\'][^>]*>(.*?)</script>',+ re.DOTALL | re.IGNORECASE,+)+_NEXT_APP_FLIGHT_RE = re.compile(+ r'self\.__next_f\.push\(\s*\[\s*1\s*,\s*("(?:[^"\\]|\\.)*")\s*\]\s*\)'+)+++def parse_next_pages_data(document: str) -> dict | None:+ """Return the Pages Router's embedded ``__NEXT_DATA__`` object."""+ match = _NEXT_PAGES_DATA_RE.search(document)+ if not match:+ return None++ payload = match.group(1)+ for candidate in (payload, html.unescape(payload)):+ try:+ data = json.loads(candidate)+ except json.JSONDecodeError:+ continue+ return data if isinstance(data, dict) else None++ return None+++def iter_next_app_flight_chunks(document: str) -> Iterator[str]:+ """Yield decoded App Router Flight chunks embedded in a document."""+ for payload in _NEXT_APP_FLIGHT_RE.findall(document):+ try:+ chunk = json.loads(payload)+ except json.JSONDecodeError:+ continue+ if isinstance(chunk, str):+ yield chunk+++def parse_next_pages_redirect(page_props: dict) -> tuple[str, int] | None:+ """Return a Pages Router JSON redirect as ``(location, status)``."""+ location = page_props.get("__N_REDIRECT")+ status = page_props.get("__N_REDIRECT_STATUS")+ if isinstance(location, str) and type(status) is int:+ return location, status+ return Nonediff --git a/user_scanner/user_scan/creator/linktree.py b/user_scanner/user_scan/creator/linktree.pyindex 82586ee..6809b35 100644--- a/user_scanner/user_scan/creator/linktree.py+++ b/user_scanner/user_scan/creator/linktree.py@@ -1,7 +1,7 @@-import jsonimport refrom user_scanner.core.impersonate import impersonate_validate+from user_scanner.core.nextjs import parse_next_pages_datafrom user_scanner.core.result import Result@@ -11,18 +11,8 @@ def validate_linktree(user: str) -> Result:def process(response):html = response.text- # Try to parse __NEXT_DATA__ first for deep extraction- data = {}- page_props = {}- next_data_match = re.search(- r'<script[^>]*id="__NEXT_DATA__"[^>]*>(.*?)</script>', html, re.DOTALL- )- if next_data_match:- try:- data = json.loads(next_data_match.group(1))- page_props = data.get("props", {}).get("pageProps", {})- except json.JSONDecodeError:- pass+ data = parse_next_pages_data(html) or {}+ page_props = data.get("props", {}).get("pageProps", {})if response.status_code == 404:if (diff --git a/user_scanner/user_scan/creator/magnific.py b/user_scanner/user_scan/creator/magnific.pyindex bd4f0ea..e08854b 100644--- a/user_scanner/user_scan/creator/magnific.py+++ b/user_scanner/user_scan/creator/magnific.py@@ -1,7 +1,5 @@-import json-import re-from user_scanner.core.impersonate import impersonate_validate+from user_scanner.core.nextjs import parse_next_pages_datafrom user_scanner.core.orchestrator import Result@@ -13,15 +11,10 @@ def validate_magnific(user: str) -> Result:return Result.error(f"Unexpected response status: {response.status_code}")try:- match = re.search(- r'<script[^>]*id="__NEXT_DATA__"[^>]*>(.*?)</script>',- response.text,- re.DOTALL,- )- data = json.loads(match.group(1))+ data = parse_next_pages_data(response.text) or {}page = data.get("page")author = data["props"]["pageProps"].get("author")- except (AttributeError, json.JSONDecodeError, KeyError, TypeError):+ except (AttributeError, KeyError, TypeError):return Result.error("Invalid Magnific profile data")if response.status_code == 404:diff --git a/user_scanner/user_scan/dev/codecademy.py b/user_scanner/user_scan/dev/codecademy.pyindex 05ff218..df618b9 100644--- a/user_scanner/user_scan/dev/codecademy.py+++ b/user_scanner/user_scan/dev/codecademy.py@@ -1,6 +1,6 @@-from user_scanner.core.orchestrator import generic_validate, Result-import json-import re+from user_scanner.core.nextjs import parse_next_pages_data+from user_scanner.core.orchestrator import Result, generic_validate+def validate_codecademy(user):url = f"https://www.codecademy.com/profiles/{user}"@@ -14,30 +14,27 @@ def validate_codecademy(user):return Result.available()elif response.status_code == 200:- try:- match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)- if match:- data = json.loads(match.group(1))- user_data = data.get("props", {}).get("pageProps", {}).get("profile", {})- if user_data and user_data.get("username"):- extra = {"is_private": False}- if user_data.get("name"):- extra["name"] = user_data.get("name")- if user_data.get("bio"):- extra["bio"] = user_data.get("bio")- if user_data.get("location"):- extra["location"] = user_data.get("location")- if user_data.get("createdAt"):- extra["joined"] = user_data.get("createdAt")- return Result.taken(extra=extra)- else:- return Result.available()-- if f"profiles/{user}" in response.text or user in response.text:- return Result.taken(extra={"is_private": False})+ data = parse_next_pages_data(response.text)+ if data:+ user_data = (+ data.get("props", {}).get("pageProps", {}).get("profile", {})+ )+ if user_data and user_data.get("username"):+ extra = {"is_private": False}+ if user_data.get("name"):+ extra["name"] = user_data.get("name")+ if user_data.get("bio"):+ extra["bio"] = user_data.get("bio")+ if user_data.get("location"):+ extra["location"] = user_data.get("location")+ if user_data.get("createdAt"):+ extra["joined"] = user_data.get("createdAt")+ return Result.taken(extra=extra)return Result.available()- except Exception:- return Result.error("200 response status with no recognizable data, report it via GitHub issues")++ if f"profiles/{user}" in response.text or user in response.text:+ return Result.taken(extra={"is_private": False})+ return Result.available()return Result.error("Unexpected response body, report it via GitHub issues.")diff --git a/user_scanner/user_scan/dev/daily_dev.py b/user_scanner/user_scan/dev/daily_dev.pyindex 88f6971..92f3782 100644--- a/user_scanner/user_scan/dev/daily_dev.py+++ b/user_scanner/user_scan/dev/daily_dev.py@@ -1,30 +1,9 @@-import json-import re-from user_scanner.core.helpers import get_random_user_agent+from user_scanner.core.nextjs import parse_next_pages_datafrom user_scanner.core.orchestrator import generic_validatefrom user_scanner.core.result import Result-NEXT_DATA_PATTERN = re.compile(- r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',- re.DOTALL,-)---def _extract_next_data(html: str) -> dict | None:- match = NEXT_DATA_PATTERN.search(html)- if not match:- return None-- try:- data = json.loads(match.group(1))- except json.JSONDecodeError:- return None-- return data if isinstance(data, dict) else None--def validate_daily_dev(user):url = f"https://daily.dev/{user}"show_url = url@@ -39,7 +18,7 @@ def validate_daily_dev(user):if response.status_code == 404:return Result.available()- next_data = _extract_next_data(response.text)+ next_data = parse_next_pages_data(response.text)if next_data is None:return Result.error("Could not read __NEXT_DATA__ payload, report it via GitHub issues."diff --git a/user_scanner/user_scan/donation/throne.py b/user_scanner/user_scan/donation/throne.pynew file mode 100644index 0000000..ea87472--- /dev/null+++ b/user_scanner/user_scan/donation/throne.py@@ -0,0 +1,98 @@+import re+from datetime import datetime, timezone++from user_scanner.core.impersonate import impersonate_request, impersonate_validate+from user_scanner.core.nextjs import (+ parse_next_pages_data,+ parse_next_pages_redirect,+)+from user_scanner.core.result import Result++LANDING_MARKER = '<meta property="og:url" content="https://throne.com/landing">'+USERNAME_RE = re.compile(r"^[A-Za-z0-9_-]{3,50}$")+++def validate_throne(user: str) -> Result:+ if not USERNAME_RE.fullmatch(user):+ return Result.error(+ "Username must be 3-50 letters, numbers, hyphens, or underscores."+ )++ url = f"https://throne.com/{user}"++ def process(response):+ if response.status_code == 307 and response.headers.get("location") == "/":+ return Result.available()++ if response.status_code != 200:+ return Result.error(f"Unexpected response status: {response.status_code}")++ if LANDING_MARKER in response.text:+ return Result.available()++ data = parse_next_pages_data(response.text) or {}+ page_props = data["props"]["pageProps"]+ # An uncached slug returns an empty shell before Next.js resolves it.+ # Fetch the final profile or not-found result as the browser does.+ if data.get("isFallback"):+ data_response = impersonate_request(+ f"https://throne.com/_next/data/{data['buildId']}/{user}.json",+ headers={"x-nextjs-data": "1"},+ )+ data_response.raise_for_status()+ page_props = data_response.json()["pageProps"]+ # Missing profiles are 307 home redirects in 200 JSON, not HTTP redirects.+ if parse_next_pages_redirect(page_props) == ("/", 307):+ return Result.available()++ fallback = page_props.get("fallback") or {}+ profile = fallback.get(f"$sub$public/useCreatorByUsername/{user}", {})+ if profile.get("username") != user:+ return Result.error("Profile markers did not match the requested username")++ return Result.taken(extra=_extract(profile), media=_media(profile))++ return impersonate_validate(url, process)+++def _extract(profile: dict) -> dict:+ extra: dict[str, object] = {"username": profile["username"]}+ for source, target in (+ ("_id", "id"),+ ("displayName", "fullname"),+ ("bio", "bio"),+ ("creatorType", "creator_type"),+ ("mainContentPlatform", "main_platform"),+ ("isPartner", "partner"),+ ("isNSFW", "nsfw"),+ ):+ if source in profile and profile[source] not in (None, ""):+ extra[target] = profile[source]++ for source, target in (("createdAt", "joined"), ("updatedAt", "updated")):+ timestamp = profile.get(source)+ if isinstance(timestamp, (int, float)) and not isinstance(timestamp, bool):+ extra[target] = datetime.fromtimestamp(+ timestamp / 1000, timezone.utc+ ).isoformat(timespec="milliseconds")++ interests = profile.get("interests")+ if isinstance(interests, list) and interests:+ extra["interests"] = interests++ for link in profile.get("socialLinks") or []:+ if isinstance(link, dict) and link.get("type") and link.get("url"):+ extra[link["type"]] = link["url"]++ return extra+++def _media(profile: dict) -> dict:+ return {+ target: profile[source]+ for source, target in (+ ("pictureUrl", "avatar"),+ ("backgroundPictureUrl", "banner"),+ )+ if profile.get(source)+ }diff --git a/user_scanner/user_scan/gaming/nexusmods.py b/user_scanner/user_scan/gaming/nexusmods.pyindex ecd4a81..84b1971 100644--- a/user_scanner/user_scan/gaming/nexusmods.py+++ b/user_scanner/user_scan/gaming/nexusmods.py@@ -4,6 +4,7 @@ import refrom urllib.parse import quotefrom user_scanner.core.impersonate import impersonate_validate+from user_scanner.core.nextjs import iter_next_app_flight_chunksfrom user_scanner.core.result import Result@@ -58,11 +59,14 @@ def validate_nexusmods(user: str) -> Result:extra[field] = match.group(1)try:- match = re.search(- r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*UserByName(?:[^"\\]|\\.)*")\]\)</script>',- response.text,+ flight = next(+ (+ chunk+ for chunk in iter_next_app_flight_chunks(response.text)+ if "UserByName" in chunk+ ),+ "",)- flight = json.loads(match.group(1))member_id = flight.index('"memberId":')start = flight.rfind('"data":', 0, member_id) + len('"data":')profile = json.JSONDecoder().raw_decode(flight[start:])[0]diff --git a/user_scanner/user_scan/shopping/yaga_co_za.py b/user_scanner/user_scan/shopping/yaga_co_za.pyindex 9f2dfc7..099c914 100644--- a/user_scanner/user_scan/shopping/yaga_co_za.py+++ b/user_scanner/user_scan/shopping/yaga_co_za.py@@ -1,13 +1,10 @@-import html-import json-import refrom urllib.parse import quotefrom user_scanner.core.helpers import get_random_user_agent+from user_scanner.core.nextjs import parse_next_pages_datafrom user_scanner.core.orchestrator import generic_validatefrom user_scanner.core.result import Result-GEO_BLOCK_MARKER = "configured to block access from your country"@@ -35,18 +32,9 @@ def validate_yaga_co_za(user: str) -> Result:f"Unexpected response status: {response.status_code}",)- match = re.search(- r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',- response.text,- re.DOTALL,- )- if not match:- return Result.error("Could not find Next.js data")-- try:- data = json.loads(html.unescape(match.group(1)))- except json.JSONDecodeError:- return Result.error("Could not parse Next.js data")+ data = parse_next_pages_data(response.text)+ if data is None:+ return Result.error("Could not read Next.js data")page_props = data.get("props", {}).get("pageProps", {})shop = page_props.get("initialShop")diff --git a/user_scanner/user_scan/shopping/yaga_ee.py b/user_scanner/user_scan/shopping/yaga_ee.pyindex 64e2499..1a32855 100644--- a/user_scanner/user_scan/shopping/yaga_ee.py+++ b/user_scanner/user_scan/shopping/yaga_ee.py@@ -1,13 +1,10 @@-import html-import json-import refrom urllib.parse import quotefrom user_scanner.core.helpers import get_random_user_agent+from user_scanner.core.nextjs import parse_next_pages_datafrom user_scanner.core.orchestrator import generic_validatefrom user_scanner.core.result import Result-GEO_BLOCK_MARKER = "configured to block access from your country"@@ -35,18 +32,9 @@ def validate_yaga_ee(user: str) -> Result:f"Unexpected response status: {response.status_code}",)- match = re.search(- r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',- response.text,- re.DOTALL,- )- if not match:- return Result.error("Could not find Next.js data")-- try:- data = json.loads(html.unescape(match.group(1)))- except json.JSONDecodeError:- return Result.error("Could not parse Next.js data")+ data = parse_next_pages_data(response.text)+ if data is None:+ return Result.error("Could not read Next.js data")page_props = data.get("props", {}).get("pageProps", {})shop = page_props.get("initialShop")diff --git a/user_scanner/user_scan/social/buzzfeed.py b/user_scanner/user_scan/social/buzzfeed.pyindex 205000b..f7fc25e 100644--- a/user_scanner/user_scan/social/buzzfeed.py+++ b/user_scanner/user_scan/social/buzzfeed.py@@ -1,8 +1,7 @@-import json-import refrom datetime import datetime, timezonefrom user_scanner.core.impersonate import impersonate_validate+from user_scanner.core.nextjs import parse_next_pages_datafrom user_scanner.core.result import Result@@ -16,7 +15,8 @@ def validate_buzzfeed(user: str) -> Result:if r.status_code != 200:return Result.error(f"HTTP {r.status_code}")- page_props = _page_props(r.text)+ data = parse_next_pages_data(r.text) or {}+ page_props = data.get("props", {}).get("pageProps", {})user_data = page_props.get("user") or {}# Section pages (/quizzes, /tasty, /search) answer 200 with the same@@ -29,17 +29,6 @@ def validate_buzzfeed(user: str) -> Result:return impersonate_validate(url, process, allow_redirects=True)--def _page_props(text: str) -> dict:- match = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', text, re.DOTALL)- if not match:- return {}- try:- return json.loads(match.group(1)).get("props", {}).get("pageProps", {})- except json.JSONDecodeError:- return {}--def _extract(page_props: dict, user_data: dict) -> tuple[dict, dict]:extra: dict = {}media: dict = {}diff --git a/user_scanner/user_scan/social/snapchat.py b/user_scanner/user_scan/social/snapchat.pyindex 00c8e45..e1e1fcb 100644--- a/user_scanner/user_scan/social/snapchat.py+++ b/user_scanner/user_scan/social/snapchat.py@@ -1,5 +1,6 @@from user_scanner.core.helpers import get_random_user_agent-from user_scanner.core.orchestrator import generic_validate, Result+from user_scanner.core.nextjs import parse_next_pages_data+from user_scanner.core.orchestrator import Result, generic_validatedef validate_snapchat(user):@@ -28,21 +29,18 @@ def validate_snapchat(user):elif response.status_code == 200:extra = {}media = {}- try:- import re as local_re- import json as local_json- m = local_re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', response.text)- if m:- next_data = local_json.loads(m.group(1))- user_profile = next_data.get("props", {}).get("pageProps", {}).get("userProfile", {})- u_info = user_profile.get("userInfo", {})- if u_info:- if u_info.get("displayName"):- extra["display_name"] = u_info.get("displayName")- if u_info.get("snapcodeImageUrl"):- media["snapcode"] = u_info.get("snapcodeImageUrl")- except Exception:- pass+ next_data = parse_next_pages_data(response.text) or {}+ user_profile = (+ next_data.get("props", {})+ .get("pageProps", {})+ .get("userProfile", {})+ )+ u_info = user_profile.get("userInfo", {})+ if u_info:+ if u_info.get("displayName"):+ extra["display_name"] = u_info.get("displayName")+ if u_info.get("snapcodeImageUrl"):+ media["snapcode"] = u_info.get("snapcodeImageUrl")return Result.taken(extra=extra, media=media)else:return Result.error(f"HTTP {response.status_code}")diff --git a/user_scanner/user_scan/social/virgool.py b/user_scanner/user_scan/social/virgool.pyindex 9b8be10..099e3dc 100644--- a/user_scanner/user_scan/social/virgool.py+++ b/user_scanner/user_scan/social/virgool.py@@ -1,37 +1,40 @@import re-import json+from user_scanner.core.helpers import get_random_user_agent-from user_scanner.core.orchestrator import Result, make_request+from user_scanner.core.nextjs import iter_next_app_flight_chunks+from user_scanner.core.orchestrator import Result, generic_validate+def validate_virgool(user):url = f"https://virgool.io/@{user}"- show_url = urlheaders = {"User-Agent": get_random_user_agent()}- try:- response = make_request(url, headers=headers)+ def process(response):if response.status_code == 200:extra = {}- match = re.search(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*\\"followersCount\\"(?:[^"\\]|\\.)*")\]', response.text)- if match:- try:- raw = match.group(1)- unquoted = json.loads(raw)-- fc_match = re.search(r'"followersCount":(\d+)', unquoted)- if fc_match: extra['follower_count'] = int(fc_match.group(1))-- name_match = re.search(r'"name":"([^"]+)"', unquoted)- if name_match: extra['fullname'] = name_match.group(1)-- bio_match = re.search(r'"bio":"([^"]+)"', unquoted)- if bio_match: extra['bio'] = bio_match.group(1)- except Exception:- pass- return Result.taken(extra=extra, url=show_url)- elif response.status_code == 404:- return Result.available(url=show_url)- else:- return Result.error(f"Unexpected status: {response.status_code}", url=show_url)- except Exception as e:- return Result.error(e, url=show_url)+ flight = next(+ (+ chunk+ for chunk in iter_next_app_flight_chunks(response.text)+ if '"followersCount"' in chunk+ ),+ "",+ )++ fc_match = re.search(r'"followersCount":(\d+)', flight)+ if fc_match:+ extra["follower_count"] = int(fc_match.group(1))++ name_match = re.search(r'"name":"([^"]+)"', flight)+ if name_match:+ extra["fullname"] = name_match.group(1)++ bio_match = re.search(r'"bio":"([^"]+)"', flight)+ if bio_match:+ extra["bio"] = bio_match.group(1)+ return Result.taken(extra=extra)+ if response.status_code == 404:+ return Result.available()+ return Result.error(f"Unexpected status: {response.status_code}")++ return generic_validate(url, process, show_url=url, headers=headers)