Skip to content

🕵️‍♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.

feat: Next.js helper functions (#583)

Kristo Isbergd1f5bbaad374
diff --git a/tests/test_nextjs.py b/tests/test_nextjs.py
new file mode 100644
index 0000000..7fafc3b
--- /dev/null
+++ b/tests/test_nextjs.py
@@ -0,0 +1,41 @@
+import json
+
+from user_scanner.core.nextjs import (
+ iter_next_app_flight_chunks,
+ parse_next_pages_data,
+ parse_next_pages_redirect,
+)
+
+
+def test_parse_next_pages_data():
+ document = """
+ <script nonce="abc" id="__NEXT_DATA__" type="application/json">
+ {"props":{"pageProps":{"username":"alice"}}}
+ </script>
+ """
+ assert parse_next_pages_data(document) == {
+ "props": {"pageProps": {"username": "alice"}}
+ }
+ assert parse_next_pages_data('<script id="__NEXT_DATA__">{bad}</script>') is None
+
+
+def test_parse_next_pages_data_unescapes_encoded_json():
+ document = (
+ '<script id="__NEXT_DATA__">'
+ "{&quot;props&quot;:{&quot;pageProps&quot;:{}}}"
+ "</script>"
+ )
+ assert parse_next_pages_data(document) == {"props": {"pageProps": {}}}
+
+
+def test_next_app_flight_chunks_and_pages_redirect():
+ chunks = ['0:{"followersCount":7}', '1:{"name":"Alice"}']
+ document = "".join(
+ f"<script>self.__next_f.push([1,{json.dumps(chunk)}])</script>"
+ for chunk in chunks
+ )
+ assert list(iter_next_app_flight_chunks(document)) == chunks
+ assert parse_next_pages_redirect(
+ {"__N_REDIRECT": "/", "__N_REDIRECT_STATUS": 307}
+ ) == ("/", 307)
+ assert parse_next_pages_redirect({"__N_REDIRECT": "/"}) is None
diff --git a/user_scanner/core/nextjs.py b/user_scanner/core/nextjs.py
new file mode 100644
index 0000000..7b4f915
--- /dev/null
+++ b/user_scanner/core/nextjs.py
@@ -0,0 +1,49 @@
+import html
+import json
+import re
+from collections.abc import Iterator
+
+_NEXT_PAGES_DATA_RE = re.compile(
+ r'<script\b[^>]*\bid=["\']__NEXT_DATA__["\'][^>]*>(.*?)</script>',
+ re.DOTALL | re.IGNORECASE,
+)
+_NEXT_APP_FLIGHT_RE = re.compile(
+ r'self\.__next_f\.push\(\s*\[\s*1\s*,\s*("(?:[^"\\]|\\.)*")\s*\]\s*\)'
+)
+
+
+def parse_next_pages_data(document: str) -> dict | None:
+ """Return the Pages Router's embedded ``__NEXT_DATA__`` object."""
+ match = _NEXT_PAGES_DATA_RE.search(document)
+ if not match:
+ return None
+
+ payload = match.group(1)
+ for candidate in (payload, html.unescape(payload)):
+ try:
+ data = json.loads(candidate)
+ except json.JSONDecodeError:
+ continue
+ return data if isinstance(data, dict) else None
+
+ return None
+
+
+def iter_next_app_flight_chunks(document: str) -> Iterator[str]:
+ """Yield decoded App Router Flight chunks embedded in a document."""
+ for payload in _NEXT_APP_FLIGHT_RE.findall(document):
+ try:
+ chunk = json.loads(payload)
+ except json.JSONDecodeError:
+ continue
+ if isinstance(chunk, str):
+ yield chunk
+
+
+def parse_next_pages_redirect(page_props: dict) -> tuple[str, int] | None:
+ """Return a Pages Router JSON redirect as ``(location, status)``."""
+ location = page_props.get("__N_REDIRECT")
+ status = page_props.get("__N_REDIRECT_STATUS")
+ if isinstance(location, str) and type(status) is int:
+ return location, status
+ return None
diff --git a/user_scanner/user_scan/creator/linktree.py b/user_scanner/user_scan/creator/linktree.py
index 82586ee..6809b35 100644
--- a/user_scanner/user_scan/creator/linktree.py
+++ b/user_scanner/user_scan/creator/linktree.py
@@ -1,7 +1,7 @@
-import json
import re
from user_scanner.core.impersonate import impersonate_validate
+from user_scanner.core.nextjs import parse_next_pages_data
from user_scanner.core.result import Result
@@ -11,18 +11,8 @@ def validate_linktree(user: str) -> Result:
def process(response):
html = response.text
- # Try to parse __NEXT_DATA__ first for deep extraction
- data = {}
- page_props = {}
- next_data_match = re.search(
- r'<script[^>]*id="__NEXT_DATA__"[^>]*>(.*?)</script>', html, re.DOTALL
- )
- if next_data_match:
- try:
- data = json.loads(next_data_match.group(1))
- page_props = data.get("props", {}).get("pageProps", {})
- except json.JSONDecodeError:
- pass
+ data = parse_next_pages_data(html) or {}
+ page_props = data.get("props", {}).get("pageProps", {})
if response.status_code == 404:
if (
diff --git a/user_scanner/user_scan/creator/magnific.py b/user_scanner/user_scan/creator/magnific.py
index bd4f0ea..e08854b 100644
--- a/user_scanner/user_scan/creator/magnific.py
+++ b/user_scanner/user_scan/creator/magnific.py
@@ -1,7 +1,5 @@
-import json
-import re
-
from user_scanner.core.impersonate import impersonate_validate
+from user_scanner.core.nextjs import parse_next_pages_data
from user_scanner.core.orchestrator import Result
@@ -13,15 +11,10 @@ def validate_magnific(user: str) -> Result:
return Result.error(f"Unexpected response status: {response.status_code}")
try:
- match = re.search(
- r'<script[^>]*id="__NEXT_DATA__"[^>]*>(.*?)</script>',
- response.text,
- re.DOTALL,
- )
- data = json.loads(match.group(1))
+ data = parse_next_pages_data(response.text) or {}
page = data.get("page")
author = data["props"]["pageProps"].get("author")
- except (AttributeError, json.JSONDecodeError, KeyError, TypeError):
+ except (AttributeError, KeyError, TypeError):
return Result.error("Invalid Magnific profile data")
if response.status_code == 404:
diff --git a/user_scanner/user_scan/dev/codecademy.py b/user_scanner/user_scan/dev/codecademy.py
index 05ff218..df618b9 100644
--- a/user_scanner/user_scan/dev/codecademy.py
+++ b/user_scanner/user_scan/dev/codecademy.py
@@ -1,6 +1,6 @@
-from user_scanner.core.orchestrator import generic_validate, Result
-import json
-import re
+from user_scanner.core.nextjs import parse_next_pages_data
+from user_scanner.core.orchestrator import Result, generic_validate
+
def validate_codecademy(user):
url = f"https://www.codecademy.com/profiles/{user}"
@@ -14,30 +14,27 @@ def validate_codecademy(user):
return Result.available()
elif response.status_code == 200:
- try:
- match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', response.text)
- if match:
- data = json.loads(match.group(1))
- user_data = data.get("props", {}).get("pageProps", {}).get("profile", {})
- if user_data and user_data.get("username"):
- extra = {"is_private": False}
- if user_data.get("name"):
- extra["name"] = user_data.get("name")
- if user_data.get("bio"):
- extra["bio"] = user_data.get("bio")
- if user_data.get("location"):
- extra["location"] = user_data.get("location")
- if user_data.get("createdAt"):
- extra["joined"] = user_data.get("createdAt")
- return Result.taken(extra=extra)
- else:
- return Result.available()
-
- if f"profiles/{user}" in response.text or user in response.text:
- return Result.taken(extra={"is_private": False})
+ data = parse_next_pages_data(response.text)
+ if data:
+ user_data = (
+ data.get("props", {}).get("pageProps", {}).get("profile", {})
+ )
+ if user_data and user_data.get("username"):
+ extra = {"is_private": False}
+ if user_data.get("name"):
+ extra["name"] = user_data.get("name")
+ if user_data.get("bio"):
+ extra["bio"] = user_data.get("bio")
+ if user_data.get("location"):
+ extra["location"] = user_data.get("location")
+ if user_data.get("createdAt"):
+ extra["joined"] = user_data.get("createdAt")
+ return Result.taken(extra=extra)
return Result.available()
- except Exception:
- return Result.error("200 response status with no recognizable data, report it via GitHub issues")
+
+ if f"profiles/{user}" in response.text or user in response.text:
+ return Result.taken(extra={"is_private": False})
+ return Result.available()
return Result.error("Unexpected response body, report it via GitHub issues.")
diff --git a/user_scanner/user_scan/dev/daily_dev.py b/user_scanner/user_scan/dev/daily_dev.py
index 88f6971..92f3782 100644
--- a/user_scanner/user_scan/dev/daily_dev.py
+++ b/user_scanner/user_scan/dev/daily_dev.py
@@ -1,30 +1,9 @@
-import json
-import re
-
from user_scanner.core.helpers import get_random_user_agent
+from user_scanner.core.nextjs import parse_next_pages_data
from user_scanner.core.orchestrator import generic_validate
from user_scanner.core.result import Result
-NEXT_DATA_PATTERN = re.compile(
- r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
- re.DOTALL,
-)
-
-
-def _extract_next_data(html: str) -> dict | None:
- match = NEXT_DATA_PATTERN.search(html)
- if not match:
- return None
-
- try:
- data = json.loads(match.group(1))
- except json.JSONDecodeError:
- return None
-
- return data if isinstance(data, dict) else None
-
-
def validate_daily_dev(user):
url = f"https://daily.dev/{user}"
show_url = url
@@ -39,7 +18,7 @@ def validate_daily_dev(user):
if response.status_code == 404:
return Result.available()
- next_data = _extract_next_data(response.text)
+ next_data = parse_next_pages_data(response.text)
if next_data is None:
return Result.error(
"Could not read __NEXT_DATA__ payload, report it via GitHub issues."
diff --git a/user_scanner/user_scan/donation/throne.py b/user_scanner/user_scan/donation/throne.py
new file mode 100644
index 0000000..ea87472
--- /dev/null
+++ b/user_scanner/user_scan/donation/throne.py
@@ -0,0 +1,98 @@
+import re
+from datetime import datetime, timezone
+
+from user_scanner.core.impersonate import impersonate_request, impersonate_validate
+from user_scanner.core.nextjs import (
+ parse_next_pages_data,
+ parse_next_pages_redirect,
+)
+from user_scanner.core.result import Result
+
+LANDING_MARKER = '<meta property="og:url" content="https://throne.com/landing">'
+USERNAME_RE = re.compile(r"^[A-Za-z0-9_-]{3,50}$")
+
+
+def validate_throne(user: str) -> Result:
+ if not USERNAME_RE.fullmatch(user):
+ return Result.error(
+ "Username must be 3-50 letters, numbers, hyphens, or underscores."
+ )
+
+ url = f"https://throne.com/{user}"
+
+ def process(response):
+ if response.status_code == 307 and response.headers.get("location") == "/":
+ return Result.available()
+
+ if response.status_code != 200:
+ return Result.error(f"Unexpected response status: {response.status_code}")
+
+ if LANDING_MARKER in response.text:
+ return Result.available()
+
+ data = parse_next_pages_data(response.text) or {}
+ page_props = data["props"]["pageProps"]
+ # An uncached slug returns an empty shell before Next.js resolves it.
+ # Fetch the final profile or not-found result as the browser does.
+ if data.get("isFallback"):
+ data_response = impersonate_request(
+ f"https://throne.com/_next/data/{data['buildId']}/{user}.json",
+ headers={"x-nextjs-data": "1"},
+ )
+ data_response.raise_for_status()
+ page_props = data_response.json()["pageProps"]
+ # Missing profiles are 307 home redirects in 200 JSON, not HTTP redirects.
+ if parse_next_pages_redirect(page_props) == ("/", 307):
+ return Result.available()
+
+ fallback = page_props.get("fallback") or {}
+ profile = fallback.get(f"$sub$public/useCreatorByUsername/{user}", {})
+ if profile.get("username") != user:
+ return Result.error("Profile markers did not match the requested username")
+
+ return Result.taken(extra=_extract(profile), media=_media(profile))
+
+ return impersonate_validate(url, process)
+
+
+def _extract(profile: dict) -> dict:
+ extra: dict[str, object] = {"username": profile["username"]}
+ for source, target in (
+ ("_id", "id"),
+ ("displayName", "fullname"),
+ ("bio", "bio"),
+ ("creatorType", "creator_type"),
+ ("mainContentPlatform", "main_platform"),
+ ("isPartner", "partner"),
+ ("isNSFW", "nsfw"),
+ ):
+ if source in profile and profile[source] not in (None, ""):
+ extra[target] = profile[source]
+
+ for source, target in (("createdAt", "joined"), ("updatedAt", "updated")):
+ timestamp = profile.get(source)
+ if isinstance(timestamp, (int, float)) and not isinstance(timestamp, bool):
+ extra[target] = datetime.fromtimestamp(
+ timestamp / 1000, timezone.utc
+ ).isoformat(timespec="milliseconds")
+
+ interests = profile.get("interests")
+ if isinstance(interests, list) and interests:
+ extra["interests"] = interests
+
+ for link in profile.get("socialLinks") or []:
+ if isinstance(link, dict) and link.get("type") and link.get("url"):
+ extra[link["type"]] = link["url"]
+
+ return extra
+
+
+def _media(profile: dict) -> dict:
+ return {
+ target: profile[source]
+ for source, target in (
+ ("pictureUrl", "avatar"),
+ ("backgroundPictureUrl", "banner"),
+ )
+ if profile.get(source)
+ }
diff --git a/user_scanner/user_scan/gaming/nexusmods.py b/user_scanner/user_scan/gaming/nexusmods.py
index ecd4a81..84b1971 100644
--- a/user_scanner/user_scan/gaming/nexusmods.py
+++ b/user_scanner/user_scan/gaming/nexusmods.py
@@ -4,6 +4,7 @@ import re
from urllib.parse import quote
from user_scanner.core.impersonate import impersonate_validate
+from user_scanner.core.nextjs import iter_next_app_flight_chunks
from user_scanner.core.result import Result
@@ -58,11 +59,14 @@ def validate_nexusmods(user: str) -> Result:
extra[field] = match.group(1)
try:
- match = re.search(
- r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*UserByName(?:[^"\\]|\\.)*")\]\)</script>',
- response.text,
+ flight = next(
+ (
+ chunk
+ for chunk in iter_next_app_flight_chunks(response.text)
+ if "UserByName" in chunk
+ ),
+ "",
)
- flight = json.loads(match.group(1))
member_id = flight.index('"memberId":')
start = flight.rfind('"data":', 0, member_id) + len('"data":')
profile = json.JSONDecoder().raw_decode(flight[start:])[0]
diff --git a/user_scanner/user_scan/shopping/yaga_co_za.py b/user_scanner/user_scan/shopping/yaga_co_za.py
index 9f2dfc7..099c914 100644
--- a/user_scanner/user_scan/shopping/yaga_co_za.py
+++ b/user_scanner/user_scan/shopping/yaga_co_za.py
@@ -1,13 +1,10 @@
-import html
-import json
-import re
from urllib.parse import quote
from user_scanner.core.helpers import get_random_user_agent
+from user_scanner.core.nextjs import parse_next_pages_data
from user_scanner.core.orchestrator import generic_validate
from user_scanner.core.result import Result
-
GEO_BLOCK_MARKER = "configured to block access from your country"
@@ -35,18 +32,9 @@ def validate_yaga_co_za(user: str) -> Result:
f"Unexpected response status: {response.status_code}",
)
- match = re.search(
- r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
- response.text,
- re.DOTALL,
- )
- if not match:
- return Result.error("Could not find Next.js data")
-
- try:
- data = json.loads(html.unescape(match.group(1)))
- except json.JSONDecodeError:
- return Result.error("Could not parse Next.js data")
+ data = parse_next_pages_data(response.text)
+ if data is None:
+ return Result.error("Could not read Next.js data")
page_props = data.get("props", {}).get("pageProps", {})
shop = page_props.get("initialShop")
diff --git a/user_scanner/user_scan/shopping/yaga_ee.py b/user_scanner/user_scan/shopping/yaga_ee.py
index 64e2499..1a32855 100644
--- a/user_scanner/user_scan/shopping/yaga_ee.py
+++ b/user_scanner/user_scan/shopping/yaga_ee.py
@@ -1,13 +1,10 @@
-import html
-import json
-import re
from urllib.parse import quote
from user_scanner.core.helpers import get_random_user_agent
+from user_scanner.core.nextjs import parse_next_pages_data
from user_scanner.core.orchestrator import generic_validate
from user_scanner.core.result import Result
-
GEO_BLOCK_MARKER = "configured to block access from your country"
@@ -35,18 +32,9 @@ def validate_yaga_ee(user: str) -> Result:
f"Unexpected response status: {response.status_code}",
)
- match = re.search(
- r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
- response.text,
- re.DOTALL,
- )
- if not match:
- return Result.error("Could not find Next.js data")
-
- try:
- data = json.loads(html.unescape(match.group(1)))
- except json.JSONDecodeError:
- return Result.error("Could not parse Next.js data")
+ data = parse_next_pages_data(response.text)
+ if data is None:
+ return Result.error("Could not read Next.js data")
page_props = data.get("props", {}).get("pageProps", {})
shop = page_props.get("initialShop")
diff --git a/user_scanner/user_scan/social/buzzfeed.py b/user_scanner/user_scan/social/buzzfeed.py
index 205000b..f7fc25e 100644
--- a/user_scanner/user_scan/social/buzzfeed.py
+++ b/user_scanner/user_scan/social/buzzfeed.py
@@ -1,8 +1,7 @@
-import json
-import re
from datetime import datetime, timezone
from user_scanner.core.impersonate import impersonate_validate
+from user_scanner.core.nextjs import parse_next_pages_data
from user_scanner.core.result import Result
@@ -16,7 +15,8 @@ def validate_buzzfeed(user: str) -> Result:
if r.status_code != 200:
return Result.error(f"HTTP {r.status_code}")
- page_props = _page_props(r.text)
+ data = parse_next_pages_data(r.text) or {}
+ page_props = data.get("props", {}).get("pageProps", {})
user_data = page_props.get("user") or {}
# Section pages (/quizzes, /tasty, /search) answer 200 with the same
@@ -29,17 +29,6 @@ def validate_buzzfeed(user: str) -> Result:
return impersonate_validate(url, process, allow_redirects=True)
-
-def _page_props(text: str) -> dict:
- match = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', text, re.DOTALL)
- if not match:
- return {}
- try:
- return json.loads(match.group(1)).get("props", {}).get("pageProps", {})
- except json.JSONDecodeError:
- return {}
-
-
def _extract(page_props: dict, user_data: dict) -> tuple[dict, dict]:
extra: dict = {}
media: dict = {}
diff --git a/user_scanner/user_scan/social/snapchat.py b/user_scanner/user_scan/social/snapchat.py
index 00c8e45..e1e1fcb 100644
--- a/user_scanner/user_scan/social/snapchat.py
+++ b/user_scanner/user_scan/social/snapchat.py
@@ -1,5 +1,6 @@
from user_scanner.core.helpers import get_random_user_agent
-from user_scanner.core.orchestrator import generic_validate, Result
+from user_scanner.core.nextjs import parse_next_pages_data
+from user_scanner.core.orchestrator import Result, generic_validate
def validate_snapchat(user):
@@ -28,21 +29,18 @@ def validate_snapchat(user):
elif response.status_code == 200:
extra = {}
media = {}
- try:
- import re as local_re
- import json as local_json
- m = local_re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', response.text)
- if m:
- next_data = local_json.loads(m.group(1))
- user_profile = next_data.get("props", {}).get("pageProps", {}).get("userProfile", {})
- u_info = user_profile.get("userInfo", {})
- if u_info:
- if u_info.get("displayName"):
- extra["display_name"] = u_info.get("displayName")
- if u_info.get("snapcodeImageUrl"):
- media["snapcode"] = u_info.get("snapcodeImageUrl")
- except Exception:
- pass
+ next_data = parse_next_pages_data(response.text) or {}
+ user_profile = (
+ next_data.get("props", {})
+ .get("pageProps", {})
+ .get("userProfile", {})
+ )
+ u_info = user_profile.get("userInfo", {})
+ if u_info:
+ if u_info.get("displayName"):
+ extra["display_name"] = u_info.get("displayName")
+ if u_info.get("snapcodeImageUrl"):
+ media["snapcode"] = u_info.get("snapcodeImageUrl")
return Result.taken(extra=extra, media=media)
else:
return Result.error(f"HTTP {response.status_code}")
diff --git a/user_scanner/user_scan/social/virgool.py b/user_scanner/user_scan/social/virgool.py
index 9b8be10..099e3dc 100644
--- a/user_scanner/user_scan/social/virgool.py
+++ b/user_scanner/user_scan/social/virgool.py
@@ -1,37 +1,40 @@
import re
-import json
+
from user_scanner.core.helpers import get_random_user_agent
-from user_scanner.core.orchestrator import Result, make_request
+from user_scanner.core.nextjs import iter_next_app_flight_chunks
+from user_scanner.core.orchestrator import Result, generic_validate
+
def validate_virgool(user):
url = f"https://virgool.io/@{user}"
- show_url = url
headers = {"User-Agent": get_random_user_agent()}
- try:
- response = make_request(url, headers=headers)
+ def process(response):
if response.status_code == 200:
extra = {}
- match = re.search(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*\\"followersCount\\"(?:[^"\\]|\\.)*")\]', response.text)
- if match:
- try:
- raw = match.group(1)
- unquoted = json.loads(raw)
-
- fc_match = re.search(r'"followersCount":(\d+)', unquoted)
- if fc_match: extra['follower_count'] = int(fc_match.group(1))
-
- name_match = re.search(r'"name":"([^"]+)"', unquoted)
- if name_match: extra['fullname'] = name_match.group(1)
-
- bio_match = re.search(r'"bio":"([^"]+)"', unquoted)
- if bio_match: extra['bio'] = bio_match.group(1)
- except Exception:
- pass
- return Result.taken(extra=extra, url=show_url)
- elif response.status_code == 404:
- return Result.available(url=show_url)
- else:
- return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
- except Exception as e:
- return Result.error(e, url=show_url)
+ flight = next(
+ (
+ chunk
+ for chunk in iter_next_app_flight_chunks(response.text)
+ if '"followersCount"' in chunk
+ ),
+ "",
+ )
+
+ fc_match = re.search(r'"followersCount":(\d+)', flight)
+ if fc_match:
+ extra["follower_count"] = int(fc_match.group(1))
+
+ name_match = re.search(r'"name":"([^"]+)"', flight)
+ if name_match:
+ extra["fullname"] = name_match.group(1)
+
+ bio_match = re.search(r'"bio":"([^"]+)"', flight)
+ if bio_match:
+ extra["bio"] = bio_match.group(1)
+ return Result.taken(extra=extra)
+ if response.status_code == 404:
+ return Result.available()
+ return Result.error(f"Unexpected status: {response.status_code}")
+
+ return generic_validate(url, process, show_url=url, headers=headers)