Skip to content

🕵️‍♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.

feat(user_scan): add ResPaper username module (#673)

Kristo Isberg4c36a074083a
diff --git a/user_scanner/user_scan/learning/respaper.py b/user_scanner/user_scan/learning/respaper.py
new file mode 100644
index 0000000..a10746c
--- /dev/null
+++ b/user_scanner/user_scan/learning/respaper.py
@@ -0,0 +1,252 @@
+import html
+import re
+from urllib.parse import quote, urljoin
+
+import httpx
+
+from user_scanner.core.orchestrator import generic_validate, make_request
+from user_scanner.core.result import Result
+
+BASE_URL = "https://www.respaper.com"
+
+
+def validate_respaper(user: str) -> Result:
+ encoded_user = quote(user, safe="")
+ url = f"{BASE_URL}/{encoded_user}/about"
+
+ def process(response) -> Result:
+ if response.status_code == 404:
+ if "The requested page was not found." in response.text:
+ return Result.available()
+ return Result.error("Unexpected 404 response")
+
+ canonical = re.search(
+ r'<link rel="canonical" href="([^"]+)"', response.text
+ )
+ if response.status_code == 200 and canonical:
+ if canonical.group(1).lower() != f"/{encoded_user}/about".lower():
+ return Result.error("Profile resolved to a different username")
+
+ extra = _extract_profile(response.text)
+ media = {}
+
+ avatar = re.search(
+ r'<td[^>]+class="user_info_container"[^>]*>\s*<img[^>]+src="([^"]+)"',
+ response.text,
+ )
+ if avatar and not avatar.group(1).endswith("/p.jpg"):
+ avatar_url = urljoin(url, avatar.group(1))
+ try:
+ avatar_response = make_request(
+ avatar_url, method="HEAD", follow_redirects=True
+ )
+ except httpx.HTTPError:
+ pass
+ else:
+ # ResPaper serves its shared default GIF from some user-specific
+ # .jpg URLs; uploaded profile photos are normalized to JPEG.
+ if (
+ avatar_response.status_code == 200
+ and avatar_response.headers.get("content-type") == "image/jpeg"
+ ):
+ media["avatar"] = avatar_url
+
+ return Result.taken(extra=extra, media=media)
+
+ return Result.error(f"Unexpected response status: {response.status_code}")
+
+ return generic_validate(url, process, show_url=url, follow_redirects=True)
+
+
+def _extract_profile(page: str) -> dict[str, object]:
+ extra: dict[str, object] = {}
+ name = re.search(
+ r'<h1[^>]*class="[^"]*\bDispDesc\b[^"]*"[^>]*>'
+ r"(.*?) &minus; Profile</h1>",
+ page,
+ re.DOTALL,
+ )
+ if name:
+ extra["fullname"] = _text(name.group(1))
+
+ for source, target, separator in (
+ ("City", "city", None),
+ ("State", "state", None),
+ ("Country", "country", None),
+ ("Organization", "organization", None),
+ ("NextDestination", "next_destination", None),
+ ("LifeGoals", "life_goals", None),
+ ("HobbiesSports", "hobbies_sports", r"(?:<br\s*/?>|\r?\n)+"),
+ ("Languages", "languages", r"[,;\r\n]+"),
+ ):
+ value = re.search(
+ rf'id="my_{source}"[^>]*>(.*?)</(?:span|pre)>', page, re.DOTALL
+ )
+ if not value:
+ continue
+ if separator and (items := _values(value.group(1), separator)):
+ extra[target] = items
+ elif not separator and (cleaned := _text(value.group(1))):
+ extra[target] = cleaned.rstrip(",") if target == "city" else cleaned
+
+ sections = _sections(page)
+ interests = [text for text, _ in _links(_section(sections, "Interested in")) if text]
+ if interests:
+ extra["exam_interests"] = interests
+
+ if education := _education(page):
+ extra["education"] = education
+
+ for heading, key in (("Class Pages", "class_pages"), ("Groups", "groups")):
+ if memberships := _memberships(_section(sections, heading)):
+ extra[key] = memberships
+
+ for heading, key in (
+ ("Faves", "faves_count"),
+ ("In Faves Of", "in_faves_of_count"),
+ ):
+ if count := re.search(rf"{heading} \((\d+)\)", page):
+ extra[key] = int(count.group(1))
+
+ recently_viewed = _links(
+ _section(sections, "Recently Viewed ResPapers"), items_only=True
+ )
+ extra["recently_viewed_count"] = sum(
+ bool(text and text != "More...") for text, _ in recently_viewed
+ )
+
+ uploads = [
+ f"{text} ({urljoin(BASE_URL, href)})"
+ for text, href in _links(
+ _section(sections, "ResPapers Uploaded by"), items_only=True
+ )
+ if text and text != "More..."
+ ]
+ if uploads:
+ extra["uploads"] = uploads
+
+ extra["recent_responses_count"] = len(
+ re.findall(
+ r'<table\b[^>]*id="resp_[^"]+"',
+ _section(sections, "Recent Responses by"),
+ )
+ )
+
+ if scores := _scores(_section(sections, "Scores")):
+ extra["scores"] = scores
+
+ return extra
+
+
+def _sections(page: str) -> list[tuple[str, str]]:
+ headings = list(
+ re.finditer(r'<p class="tdheader_sch">(.*?)</p>', page, re.DOTALL)
+ )
+ return [
+ (_text(match.group(1)).rstrip(" :"), page[match.end() : next_start])
+ for match, next_start in zip(
+ headings, [item.start() for item in headings[1:]] + [len(page)]
+ )
+ ]
+
+
+def _section(sections: list[tuple[str, str]], prefix: str) -> str:
+ return next((body for heading, body in sections if heading.startswith(prefix)), "")
+
+
+def _links(markup: str, *, items_only: bool = False) -> list[tuple[str, str]]:
+ item_filter = r'(?=[^>]*class="[^"]*\bmFont\b)' if items_only else ""
+ return [
+ (_text(label), html.unescape(href))
+ for href, label in re.findall(
+ rf'<a\b{item_filter}[^>]*href="([^"]+)"[^>]*>(.*?)</a>',
+ markup,
+ re.DOTALL,
+ )
+ ]
+
+
+def _education(page: str) -> list[dict[str, str | int]]:
+ education = []
+ for entry in re.findall(
+ r'<table[^>]+class="education_elem"[^>]*>(.*?)</table>', page, re.DOTALL
+ ):
+ item: dict[str, str | int] = {}
+ fields = {
+ "school": re.search(r'id="ed_name_\d+"[^>]*>(.*?)</span>', entry, re.DOTALL),
+ "address": re.search(r'id="ed_desc_\d+"[^>]*>(.*?)</span>', entry, re.DOTALL),
+ "comment": re.search(
+ r'id="rating_comment_span_\d+"[^>]*>(.*?)</span>', entry, re.DOTALL
+ ),
+ }
+ item.update(
+ (key, value)
+ for key, match in fields.items()
+ if match and (value := _text(match.group(1)).strip('"'))
+ )
+ if count := re.search(r">\+(\d+)</a>", entry):
+ item["school_member_count"] = int(count.group(1))
+ if rating := re.search(r'class="rateit"[^>]+data-score="(\d+)"', entry):
+ item["rating"] = int(rating.group(1))
+ if item:
+ education.append(item)
+ return education
+
+
+def _memberships(markup: str) -> list[dict[str, str]]:
+ memberships = []
+ for entry in re.findall(
+ r'<table[^>]+id="mygrp_\d+"[^>]*>(.*?)</table>', markup, re.DOTALL
+ ):
+ group = re.search(
+ r'<b>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', entry, re.DOTALL
+ )
+ if not group:
+ continue
+ item = {
+ "name": _text(group.group(2)),
+ "url": urljoin(BASE_URL, html.unescape(group.group(1))),
+ }
+ school = re.search(r'<a[^>]+href="/s/[^"]+"[^>]*>(.*?)</a>', entry, re.DOTALL)
+ if school:
+ item["school"] = _text(school.group(1))
+ memberships.append(item)
+ return memberships
+
+
+def _scores(markup: str) -> list[dict[str, str | int]]:
+ scores = []
+ for exam, table in re.findall(
+ r'<a[^>]+href="[^"]+-scp\.html"[^>]*>(.*?)</a>.*?'
+ r'<table[^>]+id="rtable"[^>]*>(.*?)</table>',
+ markup,
+ re.DOTALL,
+ ):
+ for row in re.findall(r'<tr class="row\d+">(.*?)</tr>', table, re.DOTALL):
+ mark = re.search(r"<b>([^<]+)</b>", row)
+ likes = re.search(r'id="milikes_span\d+">([^<]+)', row)
+ school = re.search(r'href="/s/[^"?]+[^>]*>(.*?)</a>', row, re.DOTALL)
+ remark = re.search(r'class="detailstd">(.*?)</td>', row, re.DOTALL)
+ score: dict[str, str | int] = {
+ "exam": _text(exam).removesuffix(" - Share Your Result!")
+ }
+ if mark:
+ score["marks"] = _text(mark.group(1))
+ if likes:
+ score["likes"] = int(_text(likes.group(1)))
+ if school:
+ score["school"] = _text(school.group(1))
+ if remark:
+ score["remarks"] = _text(remark.group(1))
+ scores.append(score)
+ return scores
+
+
+def _text(markup: str) -> str:
+ return re.sub(
+ r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", markup))
+ ).strip()
+
+
+def _values(markup: str, separator: str) -> list[str]:
+ return [value for item in re.split(separator, markup) if (value := _text(item))]