Skip to content

🕵️‍♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.

Refactor: separate media from extra (#496)

Vamato1030660ee34b
diff --git a/tests/test_result.py b/tests/test_result.py
index 061a38c..793bd6a 100644
--- a/tests/test_result.py
+++ b/tests/test_result.py
@@ -104,7 +104,7 @@ def test_result_update_empty_values():
def test_result_media():
result = Result.taken(media={"avatar": "https://example.com/a.png", " banner ": "https://example.com/b.png "})
assert result.media == {"avatar": "https://example.com/a.png", "banner": "https://example.com/b.png"}
-
+
# Test fallback ignores empty media values
result.update(media={"empty": " ", "none": None})
assert "empty" not in result.media
@@ -140,22 +140,25 @@ def test_output_formats():
category="Tech",
url="https://example.com/user",
extra={"Additional info:": "smth"},
+ media={"avatar": "https://example.com/a.png"},
)
d = res.as_dict()
assert d["url"] == "https://example.com/user"
assert d["extra"] == {"additional_info": "smth"}
+ assert d["media"] == {"avatar": "https://example.com/a.png"}
assert d["status"] == "Found"
assert (
res.to_csv()
- == "testuser,Tech,Example,Found,https://example.com/user,additional_info: smth,,"
+ == "testuser,Tech,Example,Found,https://example.com/user,additional_info: smth,avatar: https://example.com/a.png,"
)
json_std = res.to_json()
assert '"username": "testuser"' in json_std
assert '"url": "https://example.com/user"' in json_std
assert '"extra":{\n"additional_info":"smth"\n}' in json_std.replace(" ", "")
+ assert '"media":{\n"avatar":"https://example.com/a.png"\n}' in json_std.replace(" ", "")
res.update(is_email=True)
json_email = res.to_json()
@@ -319,4 +322,4 @@ def test_extra_update_preserves_bool_and_int_values():
res = Result.available()
res.update(extra={"is_verified": True, "follower_count": 42})
assert res.extra["is_verified"] is True
- assert res.extra["follower_count"] == 42
\ No newline at end of file
+ assert res.extra["follower_count"] == 42
diff --git a/user_scanner/core/pdf_generator.py b/user_scanner/core/pdf_generator.py
index cc74de2..8c6a69a 100644
--- a/user_scanner/core/pdf_generator.py
+++ b/user_scanner/core/pdf_generator.py
@@ -495,6 +495,11 @@ def generate_pdf_report(
hits_with_meta = []
for hit in hits:
meta = clean_metadata(hit.get("extra"))
+ media = hit.get("media")
+ if media and isinstance(media, dict):
+ for k, v in media.items():
+ if v:
+ meta.append((k, v))
if meta:
hits_with_meta.append((hit, meta))
diff --git a/user_scanner/core/result.py b/user_scanner/core/result.py
index cc8ea95..d8e43fa 100644
--- a/user_scanner/core/result.py
+++ b/user_scanner/core/result.py
@@ -195,26 +195,20 @@ class Result:
return json.dumps(data, indent=4)
def to_csv(self) -> str:
+ def flatten_dict(d):
+ result = ""
+ for key, value in d.items():
+ result += f"{key}: {value}; "
+ return result.rstrip("; ")
+
# uses .as_dict() since header has "username"
data = self.as_dict()
# flatten multiline extra string parameters so it doesn't break row alignments
- if data.get("extra"):
- clean_extra = ""
- for key, value in data["extra"].items():
- clean_extra += f"{key}: {value}; "
+ data["extra"] = flatten_dict(data["extra"]) if data.get("extra") else ""
+ data["media"] = flatten_dict(data["media"]) if data.get("media") else ""
- data["extra"] = clean_extra.rstrip("; ")
- else:
- data["extra"] = ""
- if data.get("media"):
- clean_media = ""
- for key, value in data["media"].items():
- clean_media += f"{key}: {value}; "
- data["media"] = clean_media.rstrip("; ")
- else:
- data["media"] = ""
del data["is_email"]
diff --git a/user_scanner/email_scan/learning/duolingo.py b/user_scanner/email_scan/learning/duolingo.py
index bd01755..3fcc92c 100644
--- a/user_scanner/email_scan/learning/duolingo.py
+++ b/user_scanner/email_scan/learning/duolingo.py
@@ -1,22 +1,23 @@
import httpx
+
from user_scanner.core.result import Result
async def _check(email: str) -> Result:
show_url = "https://duolingo.com"
headers = {
- 'authority': 'www.duolingo.com',
- 'Accept': 'application/json, text/plain, */*',
- 'Accept-Language': 'en-US,en;q=0.9',
- 'User-Agent': "Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0",
- 'Referer': 'https://www.duolingo.com/',
+ "authority": "www.duolingo.com",
+ "Accept": "application/json, text/plain, */*",
+ "Accept-Language": "en-US,en;q=0.9",
+ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0",
+ "Referer": "https://www.duolingo.com/",
}
try:
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
response = await client.get(
f"https://www.duolingo.com/2017-06-30/users?email={email}",
- headers=headers
+ headers=headers,
)
if response.status_code == 200:
@@ -25,7 +26,8 @@ async def _check(email: str) -> Result:
if data.get("users") and len(data["users"]) > 0:
user = data["users"][0]
extra = {}
-
+ media = {}
+
if user.get("name"):
extra["name"] = user["name"]
if user.get("username"):
@@ -38,14 +40,14 @@ async def _check(email: str) -> Result:
extra["has_google_id"] = user["hasGoogleId"]
if "hasFacebookId" in user:
extra["has_facebook_id"] = user["hasFacebookId"]
-
+
pic = user.get("picture")
if pic:
if pic.startswith("//"):
pic = "https:" + pic
- extra["picture"] = pic
-
- return Result.taken(url=show_url, extra=extra)
+ media["avatar"] = pic
+
+ return Result.taken(extra=extra, media=media, url=show_url)
else:
return Result.available(url=show_url)
diff --git a/user_scanner/email_scan/shopping/etsy.py b/user_scanner/email_scan/shopping/etsy.py
index 09cb114..190c213 100644
--- a/user_scanner/email_scan/shopping/etsy.py
+++ b/user_scanner/email_scan/shopping/etsy.py
@@ -91,10 +91,13 @@ async def _check(email: str) -> Result:
"privacy": f"Items are {fav_items} | Shops are {fav_shops}",
"joined": date_created,
"last profile update": date_updated,
+ }
+
+ media = {
"avatar": pfp,
}
- return Result.taken(url=show_url, extra=extras)
+ return Result.taken(url=show_url, extra=extras, media=media)
return Result.error("Unexpected response body structure")
diff --git a/user_scanner/email_scan/social/gravatar.py b/user_scanner/email_scan/social/gravatar.py
index 49131ee..906126a 100644
--- a/user_scanner/email_scan/social/gravatar.py
+++ b/user_scanner/email_scan/social/gravatar.py
@@ -2,7 +2,7 @@ import hashlib
import httpx
from user_scanner.core.result import Result
-def _extract_profile_data(entry: dict, extra: dict) -> None:
+def _extract_profile_data(entry: dict, extra: dict, media: dict) -> None:
if entry.get("preferredUsername"):
extra["username"] = str(entry["preferredUsername"]).strip()
if entry.get("displayName"):
@@ -10,7 +10,7 @@ def _extract_profile_data(entry: dict, extra: dict) -> None:
if entry.get("profileUrl"):
extra["profile_url"] = str(entry["profileUrl"]).strip()
if entry.get("thumbnailUrl"):
- extra["thumbnail_url"] = str(entry["thumbnailUrl"]).strip()
+ media["thumbnail_url"] = str(entry["thumbnailUrl"]).strip()
if entry.get("aboutMe"):
extra["bio"] = str(entry["aboutMe"]).strip()
if entry.get("currentLocation"):
@@ -32,7 +32,7 @@ def _extract_profile_data(entry: dict, extra: dict) -> None:
if isinstance(p, dict) and p.get("value") is not None and str(p["value"]).strip()
]
if photo_list:
- extra["photos"] = ", ".join(photo_list)
+ media["photos"] = ", ".join(photo_list)
accounts = entry.get("accounts")
if isinstance(accounts, list):
@@ -90,7 +90,8 @@ async def _check(email: str) -> Result:
async with httpx.AsyncClient(timeout=15.0) as client:
response = await client.get(url, headers=headers)
if response.status_code == 200:
- extra = {"avatar_url": f"https://www.gravatar.com/avatar/{email_hash}"}
+ extra: dict = {}
+ media = {"avatar": f"https://www.gravatar.com/avatar/{email_hash}"}
profile_url = f"https://en.gravatar.com/{email_hash}.json"
try:
profile_resp = await client.get(profile_url, headers=headers, timeout=15.0)
@@ -98,19 +99,20 @@ async def _check(email: str) -> Result:
data = profile_resp.json()
entries = data.get("entry", [])
if entries and isinstance(entries, list):
- _extract_profile_data(entries[0], extra)
+ _extract_profile_data(entries[0], extra, media)
except Exception:
pass
final_url = extra.get("profile_url", show_url)
- return Result.taken(url=final_url, extra=extra)
+ return Result.taken(url=final_url, extra=extra, media=media)
elif response.status_code == 404:
# Also fall back to check MD5 since some older profiles might only map via MD5
email_md5 = hashlib.md5(email_clean.encode("utf-8")).hexdigest()
url_md5 = f"https://www.gravatar.com/avatar/{email_md5}?d=404"
-
+
response_md5 = await client.get(url_md5, headers=headers)
if response_md5.status_code == 200:
- extra = {"avatar_url": f"https://www.gravatar.com/avatar/{email_md5}"}
+ extra = {}
+ media = {"avatar": f"https://www.gravatar.com/avatar/{email_md5}"}
profile_url_md5 = f"https://en.gravatar.com/{email_md5}.json"
try:
profile_resp = await client.get(profile_url_md5, headers=headers, timeout=15.0)
@@ -118,11 +120,11 @@ async def _check(email: str) -> Result:
data = profile_resp.json()
entries = data.get("entry", [])
if entries and isinstance(entries, list):
- _extract_profile_data(entries[0], extra)
+ _extract_profile_data(entries[0], extra, media)
except Exception:
pass
final_url = extra.get("profile_url", show_url)
- return Result.taken(url=final_url, extra=extra)
+ return Result.taken(url=final_url, extra=extra, media=media)
elif response_md5.status_code == 404:
return Result.available(url=show_url)
else:
diff --git a/user_scanner/user_scan/adult/admireme_vip.py b/user_scanner/user_scan/adult/admireme_vip.py
index 8dc629c..ebcb046 100644
--- a/user_scanner/user_scan/adult/admireme_vip.py
+++ b/user_scanner/user_scan/adult/admireme_vip.py
@@ -17,6 +17,7 @@ def validate_admireme_vip(user):
if "creator-stat subscriber" in response.text or response.status_code == 200:
if "creator-stat subscriber" in response.text or "admireme.vip" in response.text:
extra = {}
+ media = {}
# Extract Display Name/Title
title_match = re.search(
@@ -63,28 +64,28 @@ def validate_admireme_vip(user):
image_match = re.search(
r'<meta\s+property="og:image"\s+content="([^"]+)"', response.text, re.IGNORECASE)
if image_match:
- extra["image"] = image_match.group(1).strip()
+ media["image"] = image_match.group(1).strip()
else:
bg_video_match = re.search(
r'class="profile_video"[^>]*style="[^"]*background-image:\s*url\(\s*\'?([^\')]+)\'?\s*\)', response.text, re.IGNORECASE)
if bg_video_match:
- extra["image"] = bg_video_match.group(1).strip()
+ media["image"] = bg_video_match.group(1).strip()
else:
bg_pic_match = re.search(
r'class="profile-pic[^"]*"[^>]*style="[^"]*background-image:\s*url\(\s*\'?([^\')]+)\'?\s*\)', response.text, re.IGNORECASE)
if bg_pic_match:
- extra["image"] = bg_pic_match.group(1).strip()
+ media["image"] = bg_pic_match.group(1).strip()
else:
img_tag_match = re.search(
r'class="profile-pic[^"]*".*?<img[^>]+src="([^"]+)"', response.text, re.DOTALL | re.IGNORECASE)
if img_tag_match:
- extra["image"] = img_tag_match.group(1).strip()
+ media["image"] = img_tag_match.group(1).strip()
# Extract Banner Image
banner_match = re.search(
r'id="banner".*?<img[^>]+src="([^"]+)"', response.text, re.DOTALL | re.IGNORECASE)
if banner_match:
- extra["banner"] = banner_match.group(1).strip()
+ media["banner"] = banner_match.group(1).strip()
# Extract Subscription Price
price_match = re.search(
@@ -109,7 +110,7 @@ def validate_admireme_vip(user):
if t_count or s_count:
extra["posts"] = str(t_count + s_count)
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
if response.status_code == 404 or "<title>Page Not Found |" in response.text:
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/adult/adultism.py b/user_scanner/user_scan/adult/adultism.py
index 58f751f..db9f63a 100644
--- a/user_scanner/user_scan/adult/adultism.py
+++ b/user_scanner/user_scan/adult/adultism.py
@@ -17,13 +17,14 @@ def validate_adultism(user):
)
if is_taken:
extra = {}
+ media = {}
# Extract Avatar
avatar_match = re.search(r'<img[^>]+src="([^"]+)"[^>]*class="[^"]*profile-image', response.text, re.IGNORECASE) or \
re.search(
r'<img[^>]+class="[^"]*profile-image[^"]*"[^>]*src="([^"]+)"', response.text, re.IGNORECASE)
if avatar_match and "defaults/member" not in avatar_match.group(1):
- extra["image"] = avatar_match.group(1)
+ media["image"] = avatar_match.group(1)
# Extract Channel ID
channel_match = re.search(
@@ -135,7 +136,7 @@ def validate_adultism(user):
except Exception:
pass # Keep going if friends page request fails
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
if response.status_code == 404 or "Page not found" in response.text:
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/adult/bdsmlr.py b/user_scanner/user_scan/adult/bdsmlr.py
index 831f22f..4ece7ce 100644
--- a/user_scanner/user_scan/adult/bdsmlr.py
+++ b/user_scanner/user_scan/adult/bdsmlr.py
@@ -10,6 +10,7 @@ def validate_bdsmlr(user):
def process(response):
if response.status_code == 200 and "login" in response.text:
extra = {}
+ media = {}
# Extract blog title
title_match = re.search(
@@ -25,7 +26,7 @@ def validate_bdsmlr(user):
image_match = re.search(
r'<meta\s+property="og:image"\s+content="([^"]+)"', response.text, re.IGNORECASE)
if image_match and "default_avatar" not in image_match.group(1):
- extra["image"] = image_match.group(1).strip()
+ media["image"] = image_match.group(1).strip()
# Extract description
desc_match = re.search(
@@ -41,7 +42,7 @@ def validate_bdsmlr(user):
if blog_id_match:
extra["blog_id"] = blog_id_match.group(1)
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
if "This blog doesn't exist." in response.text or response.status_code == 404:
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/adult/bdsmsingles.py b/user_scanner/user_scan/adult/bdsmsingles.py
index 5f72e1c..058038a 100644
--- a/user_scanner/user_scan/adult/bdsmsingles.py
+++ b/user_scanner/user_scan/adult/bdsmsingles.py
@@ -9,6 +9,7 @@ def validate_bdsmsingles(user):
def process(response):
if response.status_code == 200 and "<title>Profile" in response.text:
extra = {}
+ media = {}
# Headline
headline_match = re.search(r"<h1>(.*?)</h1>", response.text)
@@ -29,7 +30,7 @@ def validate_bdsmsingles(user):
avatar_match = re.search(
r'src="(https://media\.bdsmsingles\.com/images/user_photo/[^"]+)"', response.text)
if avatar_match and "nophoto" not in avatar_match.group(1):
- extra["image"] = avatar_match.group(1)
+ media["image"] = avatar_match.group(1)
# Table fields extraction
def extract_table_field(field_name):
@@ -59,7 +60,7 @@ def validate_bdsmsingles(user):
if ethnicity:
extra["ethnicity"] = ethnicity
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
if response.status_code == 302 or "BDSM Singles" in response.text:
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/adult/bentbox.py b/user_scanner/user_scan/adult/bentbox.py
index 3b62f65..a765714 100644
--- a/user_scanner/user_scan/adult/bentbox.py
+++ b/user_scanner/user_scan/adult/bentbox.py
@@ -13,6 +13,7 @@ def validate_bentbox(user):
return Result.available(url=show_url)
extra = {}
+ media = {}
# Extract profile data from OpenGraph or Title tags
title_match = re.search(
@@ -48,7 +49,7 @@ def validate_bentbox(user):
image_match = re.search(
r'<meta\s+property="og:image"\s+content="([^"]+)"', response.text, re.IGNORECASE)
if image_match:
- extra["image"] = image_match.group(1).strip()
+ media["image"] = image_match.group(1).strip()
# Extract Identifier / User ID
id_match = re.search(
@@ -84,7 +85,7 @@ def validate_bentbox(user):
if videos_match:
extra["videos"] = videos_match.group(1)
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
if response.status_code in (403, 429):
return Result.error(
diff --git a/user_scanner/user_scan/adult/e621.py b/user_scanner/user_scan/adult/e621.py
index 7ac9e80..f3ff51d 100644
--- a/user_scanner/user_scan/adult/e621.py
+++ b/user_scanner/user_scan/adult/e621.py
@@ -49,10 +49,14 @@ def validate_e621(user: str) -> Result:
if response.status_code == 404:
if _reason(response.text) == "not found":
return Result.available(url=show_url)
- return Result.error("Unexpected 404 (not the not-found payload)", url=show_url)
+ return Result.error(
+ "Unexpected 404 (not the not-found payload)", url=show_url
+ )
if response.status_code != 200:
- return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
+ return Result.error(
+ f"Unexpected status: {response.status_code}", url=show_url
+ )
try:
profile = json.loads(response.text)
@@ -64,7 +68,8 @@ def validate_e621(user: str) -> Result:
if str(profile.get("name", "")).lower() != user.lower():
return Result.error("Profile confirmation not found", url=show_url)
- return Result.taken(extra=_extract_profile(profile), url=show_url)
+ extra, media = _extract_profile(profile)
+ return Result.taken(extra=extra, media=media, url=show_url)
return impersonate_validate(url, process, show_url=show_url, headers=HEADERS)
@@ -76,8 +81,9 @@ def _reason(body: str) -> str:
return ""
-def _extract_profile(profile: dict) -> dict:
+def _extract_profile(profile: dict) -> tuple[dict, dict]:
extra = {}
+ media = {}
for key, label in FIELDS.items():
value = profile.get(key)
@@ -93,11 +99,11 @@ def _extract_profile(profile: dict) -> dict:
# The avatar is a regular post, so link it rather than reporting a bare id.
avatar_id = profile.get("avatar_id")
if avatar_id:
- extra["avatar"] = f"{BASE_URL}/posts/{avatar_id}"
+ media["avatar"] = f"{BASE_URL}/posts/{avatar_id}"
for key, label in (("profile_about", "about"), ("profile_artinfo", "artist_info")):
text = re.sub(r"\s+", " ", str(profile.get(key) or "")).strip()
if text:
extra[label] = text
- return extra
+ return extra, media
diff --git a/user_scanner/user_scan/adult/livejasmin.py b/user_scanner/user_scan/adult/livejasmin.py
index 45e5819..ad3cc02 100644
--- a/user_scanner/user_scan/adult/livejasmin.py
+++ b/user_scanner/user_scan/adult/livejasmin.py
@@ -47,8 +47,9 @@ def validate_livejasmin(user: str) -> Result:
# existence signal rather than a live-stream one.
performer = _meta(response.text, "performerName")
if performer and performer.lower() == user.lower():
- extra = {"type": "performer", **_extract_profile(response.text)}
- return Result.taken(extra=extra, url=show_url)
+ extra, media = _extract_profile(response.text)
+ extra = {"type": "performer", **extra}
+ return Result.taken(extra=extra, media=media, url=show_url)
# The miss page is titled "<handle> is not available". Not being a performer
# says nothing about member accounts, which have no public page at all, so
@@ -110,8 +111,9 @@ def _signup_token(refresh: bool = False) -> str | None:
return _form_token
-def _extract_profile(html_text: str) -> dict:
+def _extract_profile(html_text: str) -> tuple[dict, dict]:
extra = {}
+ media = {}
# The player bootstrap config holds the profile details as a flat JS object.
for key, label in CONFIG_FIELDS.items():
@@ -125,9 +127,9 @@ def _extract_profile(html_text: str) -> dict:
avatar = _meta(html_text, "og:image")
# The miss page falls back to a static site banner; only keep real snapshots.
if avatar and "seo_og" not in avatar:
- extra["avatar"] = avatar
+ media["avatar"] = avatar
- return extra
+ return extra, media
def _meta(html_text: str, name: str) -> str | None:
diff --git a/user_scanner/user_scan/adult/motherless.py b/user_scanner/user_scan/adult/motherless.py
index c4ebea4..bce60cc 100644
--- a/user_scanner/user_scan/adult/motherless.py
+++ b/user_scanner/user_scan/adult/motherless.py
@@ -24,7 +24,9 @@ def validate_motherless(user):
if 'class="not-available"' in body:
if "invalid" in body.lower():
return Result.error("Username rejected by Motherless", url=show_url)
- return Result.taken(extra=_fetch_profile(show_url), url=show_url)
+
+ extra, media = _fetch_profile(show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
if 'class="available"' in body:
return Result.available(url=show_url)
@@ -36,20 +38,21 @@ def validate_motherless(user):
)
-def _fetch_profile(profile_url: str) -> dict:
+def _fetch_profile(profile_url: str) -> tuple[dict, dict]:
"""The checkusername endpoint only reports availability, so pull the public
member page for metadata. Best-effort: a failed fetch yields no extra."""
try:
response = make_request(profile_url)
if response.status_code != 200:
- return {}
+ return {}, {}
return _extract_profile(response.text)
except Exception:
- return {}
+ return {}, {}
-def _extract_profile(html_text: str) -> dict:
+def _extract_profile(html_text: str) -> tuple[dict, dict]:
extra = {}
+ media = {}
# The public member page renders two field blocks: "profile-stats" rows
# (<span>Label:</span> value) and "profile-member-info" rows
@@ -70,6 +73,6 @@ def _extract_profile(html_text: str) -> dict:
avatar = re.search(r'og:image"\s+content="([^"]+)"', html_text, re.IGNORECASE)
if avatar:
- extra["avatar"] = avatar.group(1)
+ media["avatar"] = avatar.group(1)
- return extra
+ return extra, media
diff --git a/user_scanner/user_scan/adult/spankbang.py b/user_scanner/user_scan/adult/spankbang.py
index b48d86b..af7deca 100644
--- a/user_scanner/user_scan/adult/spankbang.py
+++ b/user_scanner/user_scan/adult/spankbang.py
@@ -19,7 +19,8 @@ def validate_spankbang(user):
# A real profile answers 200 with a "<user> Profile ... @ SpankBang"
# title; the not-found page is a 200-less generic front page.
if _is_profile(response.text, user):
- return Result.taken(extra=_extract_profile(response.text), url=show_url)
+ extra, media = _extract_profile(response.text)
+ return Result.taken(extra=extra, media=media, url=show_url)
return Result.error("Profile confirmation not found", url=show_url)
@@ -35,9 +36,9 @@ def _is_profile(html_text: str, user: str) -> bool:
return title.startswith(user.lower() + " profile") and "spankbang" in title
-def _extract_profile(html_text: str) -> dict:
+def _extract_profile(html_text: str) -> tuple[dict, dict]:
extra = {}
-
+ media = {}
# The OpenGraph tags are templated boilerplate here, so read the real
# server-rendered profile markup instead.
name = re.search(r'<h1 class="profile_name">\s*([^<]+?)\s*</h1>', html_text, re.IGNORECASE)
@@ -68,7 +69,7 @@ def _extract_profile(html_text: str) -> dict:
# cross-referencing even when it falls back to the robohash default.
avatar = re.search(r'class="avatar"[^>]*?src="([^"]+)"', html_text, re.IGNORECASE)
if avatar:
- extra["avatar"] = html.unescape(avatar.group(1).strip())
+ media["avatar"] = html.unescape(avatar.group(1).strip())
# Stat tabs render as <a href="/profile/<user>/<slug>">Label <span>N</span></a>.
# Key off the URL slug rather than the visible label so localisation never
@@ -81,4 +82,4 @@ def _extract_profile(html_text: str) -> dict:
if slug not in extra and count not in ("0", ""):
extra[slug] = count
- return extra
+ return extra, media
diff --git a/user_scanner/user_scan/adult/xnxx.py b/user_scanner/user_scan/adult/xnxx.py
index c314330..c05ed82 100644
--- a/user_scanner/user_scan/adult/xnxx.py
+++ b/user_scanner/user_scan/adult/xnxx.py
@@ -41,33 +41,36 @@ def validate_xnxx(user: str) -> Result:
if "already exists" not in message.lower():
return Result.error(message or "Name check gave no verdict", url=show_url)
- return Result.taken(extra=_public_profile(user), url=show_url)
+
+ extra, media = _public_profile(user)
+ return Result.taken(extra=extra, media=media, url=show_url)
return impersonate_validate(
CHECK_URL, process, show_url=show_url, params={"profile": user}, headers=HEADERS
)
-def _public_profile(user: str) -> dict:
+def _public_profile(user: str) -> tuple[dict, dict]:
"""Registered names only get a public page when they are a model or an
uploader; plain members have none. Best-effort: a miss just means the
account exists without anything public to show."""
try:
response = impersonate_request(f"{BASE_URL}/pornstar/{user}", allow_redirects=True)
if response.status_code != 200:
- return {"type": "member"}
+ return {"type": "member"}, {}
except Exception:
- return {"type": "member"}
+ return {"type": "member"}, {}
user_obj = _user_object(response.text)
if user_obj.get("username", "").lower() != user.lower():
- return {"type": "member"}
+ return {"type": "member"}, {}
return _extract_profile(response.text, user_obj)
-def _extract_profile(html_text: str, user_obj: dict) -> dict:
+def _extract_profile(html_text: str, user_obj: dict) -> tuple[dict, dict]:
extra = {}
+ media = {}
namespace = re.match(r"/([^/]+)/", user_obj.get("url") or "")
if namespace:
@@ -81,13 +84,13 @@ def _extract_profile(html_text: str, user_obj: dict) -> dict:
extra["gender"] = user_obj["sex"]
avatar = user_obj.get("profile_picture") or user_obj.get("profile_picture_small")
if avatar:
- extra["avatar"] = avatar
+ media["avatar"] = avatar
aliases = _aliases(html_text, user_obj.get("display", ""))
if aliases:
extra["aliases"] = ", ".join(aliases)
- return extra
+ return extra, media
def _user_object(html_text: str) -> dict:
diff --git a/user_scanner/user_scan/adult/xvideos.py b/user_scanner/user_scan/adult/xvideos.py
index 52d71ac..73fa46d 100644
--- a/user_scanner/user_scan/adult/xvideos.py
+++ b/user_scanner/user_scan/adult/xvideos.py
@@ -23,15 +23,17 @@ def validate_xvideos(user):
# A real profile is titled "<name> - Profile page - XVIDEOS.COM".
if "profile page" in _title(response.text).lower():
- return Result.taken(extra=_extract_profile(response.text), url=show_url)
+ extra, media = _extract_profile(response.text)
+ return Result.taken(extra=extra, media=media, url=show_url)
return Result.error("Profile confirmation not found", url=show_url)
return generic_validate(url, process, show_url=show_url)
-def _extract_profile(html_text: str) -> dict:
+def _extract_profile(html_text: str) -> tuple[dict, dict]:
extra = {}
+ media = {}
# Structured identity from the embedded JSON `user` object.
user = _user_object(html_text)
@@ -41,7 +43,7 @@ def _extract_profile(html_text: str) -> dict:
extra["user_id"] = str(user["id_user"])
avatar = user.get("profile_picture") or user.get("profile_picture_small")
if avatar:
- extra["avatar"] = avatar
+ media["avatar"] = avatar
# The visible "profile info" panel carries gender, age, country, hits,
# languages, signup, last activity, plus any personal/physical details the
@@ -56,7 +58,7 @@ def _extract_profile(html_text: str) -> dict:
if label and value and value.lower() != "display":
extra[label] = value
- return extra
+ return extra, media
def _user_object(html_text: str) -> dict:
diff --git a/user_scanner/user_scan/adult/youporn.py b/user_scanner/user_scan/adult/youporn.py
index 01634a1..32068ff 100644
--- a/user_scanner/user_scan/adult/youporn.py
+++ b/user_scanner/user_scan/adult/youporn.py
@@ -30,8 +30,9 @@ def validate_youporn(user: str) -> Result:
# Only a real profile echoes the requested handle in its canonical link;
# the host varies because YouPorn geo-redirects (e.g. br.youporn.com).
if response.status_code == 200 and _canonical_profile(response.text, path) == user.lower():
- extra = {"type": account_type, **_extract_profile(response.text)}
- return Result.taken(extra=extra, url=url)
+ extra, media = _extract_profile(response.text)
+ extra = {"type": account_type, **extra}
+ return Result.taken(extra=extra, media=media, url=url)
# A genuinely missing handle 404s with the dedicated not-found template;
# require its title so a blocked or transient 404 isn't read as free.
@@ -97,8 +98,9 @@ def _canonical_profile(html_text: str, path: str) -> str | None:
return match.group(1).lower() if match else None
-def _extract_profile(html_text: str) -> dict:
+def _extract_profile(html_text: str) -> tuple[dict, dict]:
extra = {}
+ media = {}
name = re.search(r'<h1 class="name-title">\s*(.*?)\s*</h1>', html_text, re.DOTALL)
if name:
@@ -121,9 +123,9 @@ def _extract_profile(html_text: str) -> dict:
avatar = re.search(
r'<div class="avatar-wrapper">.*?\bdata-src="([^"]+)"', html_text, re.DOTALL | re.IGNORECASE)
if avatar:
- extra["avatar"] = html.unescape(avatar.group(1))
+ media["avatar"] = html.unescape(avatar.group(1))
- return extra
+ return extra, media
def _title(html_text: str) -> str:
diff --git a/user_scanner/user_scan/adult/zmarsa.py b/user_scanner/user_scan/adult/zmarsa.py
index b53eeab..6165a98 100644
--- a/user_scanner/user_scan/adult/zmarsa.py
+++ b/user_scanner/user_scan/adult/zmarsa.py
@@ -9,12 +9,13 @@ def validate_zmarsa(user):
def process(response):
if "Statystyki" in response.text:
extra = {}
+ media = {}
# Avatar image URL (if not default)
avatar_match = re.search(
r'src="(https://zmarsa.com/storage/avatar/[^"]+)"', response.text)
if avatar_match and "default.jpg" not in avatar_match.group(1):
- extra["image"] = avatar_match.group(1)
+ media["image"] = avatar_match.group(1)
# Date joined
joined_match = re.search(
@@ -53,7 +54,7 @@ def validate_zmarsa(user):
if comments_match:
extra["comments"] = comments_match.group(1).strip()
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
if "<title>Error 404 - zMarsa.com<" in response.text:
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/community/coderlegion.py b/user_scanner/user_scan/community/coderlegion.py
index f6b21fe..0f73dc6 100644
--- a/user_scanner/user_scan/community/coderlegion.py
+++ b/user_scanner/user_scan/community/coderlegion.py
@@ -11,6 +11,7 @@ def validate_coderlegion(user):
return Result.available()
if response.status_code == 200:
extra = {}
+ media = {}
html = response.text
try:
# Extract name
@@ -34,7 +35,7 @@ def validate_coderlegion(user):
avatar_url = avatar_match.group(1)
if avatar_url.startswith("../"):
avatar_url = "https://coderlegion.com/" + avatar_url.lstrip("../")
- extra["avatar"] = avatar_url.replace("&amp;", "&")
+ media["avatar"] = avatar_url.replace("&amp;", "&")
# Extract stats
points_match = re.search(r'<strong>([^<]+)</strong> Points', html)
@@ -55,7 +56,7 @@ def validate_coderlegion(user):
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error(f"Unexpected status code: {response.status_code}")
diff --git a/user_scanner/user_scan/community/discourse_meta.py b/user_scanner/user_scan/community/discourse_meta.py
index 943aab9..7fb4c39 100644
--- a/user_scanner/user_scan/community/discourse_meta.py
+++ b/user_scanner/user_scan/community/discourse_meta.py
@@ -12,6 +12,7 @@ def validate_discourse_meta(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_discourse_meta(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,10 +28,10 @@ def validate_discourse_meta(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://meta.discourse.org" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media)
+
return Result.error(f"Unexpected response status: {response.status_code}")
headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"}
diff --git a/user_scanner/user_scan/community/ghost_forum.py b/user_scanner/user_scan/community/ghost_forum.py
index 0a0844b..aaa6b17 100644
--- a/user_scanner/user_scan/community/ghost_forum.py
+++ b/user_scanner/user_scan/community/ghost_forum.py
@@ -12,6 +12,7 @@ def validate_ghost_forum(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_ghost_forum(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,10 +28,10 @@ def validate_ghost_forum(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://forum.ghost.org" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media)
+
return Result.error(f"Unexpected response status: {response.status_code}")
headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"}
diff --git a/user_scanner/user_scan/community/jupyter_forum.py b/user_scanner/user_scan/community/jupyter_forum.py
index c560226..2d8d912 100644
--- a/user_scanner/user_scan/community/jupyter_forum.py
+++ b/user_scanner/user_scan/community/jupyter_forum.py
@@ -12,6 +12,7 @@ def validate_jupyter_forum(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_jupyter_forum(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,9 +28,9 @@ def validate_jupyter_forum(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://discourse.jupyter.org" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra, url=show_url)
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media, url=show_url)
return Result.available(url=show_url)
elif response.status_code == 404:
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/community/lemmy.py b/user_scanner/user_scan/community/lemmy.py
index ab4cbbc..087f619 100644
--- a/user_scanner/user_scan/community/lemmy.py
+++ b/user_scanner/user_scan/community/lemmy.py
@@ -22,6 +22,7 @@ def validate_lemmy(user: str) -> Result:
return Result.available()
if response.status_code == 200:
extra = {}
+ media = {}
try:
data = response.json()
person_view = data.get("person_view", {})
@@ -35,7 +36,7 @@ def validate_lemmy(user: str) -> Result:
if person.get("display_name"):
extra["display_name"] = person["display_name"]
if person.get("avatar"):
- extra["avatar"] = person["avatar"]
+ media["avatar"] = person["avatar"]
if person.get("published"):
extra["joined"] = person["published"]
if "bot_account" in person:
@@ -48,7 +49,7 @@ def validate_lemmy(user: str) -> Result:
extra["comments"] = counts["comment_count"]
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error(f"Unexpected status code: {response.status_code}")
return generic_validate(url, process, show_url=show_url, timeout=5.0)
diff --git a/user_scanner/user_scan/community/stackoverflow.py b/user_scanner/user_scan/community/stackoverflow.py
index 8f312ce..abdcd56 100644
--- a/user_scanner/user_scan/community/stackoverflow.py
+++ b/user_scanner/user_scan/community/stackoverflow.py
@@ -28,6 +28,7 @@ def validate_stackoverflow(user: str) -> Result:
if found_card:
extra = {}
+ media = {}
try:
loc_match = re.search(r'<span class="user-location">([^<]*)</span>', found_card)
rep_match = re.search(r'title="[^"]*total reputation:\s*([^"]+)"', found_card)
@@ -41,10 +42,10 @@ def validate_stackoverflow(user: str) -> Result:
avatar_url = img_match.group(1).strip()
if avatar_url.startswith("//"):
avatar_url = "https:" + avatar_url
- extra["avatar"] = avatar_url.replace("&amp;", "&")
+ media["avatar"] = avatar_url.replace("&amp;", "&")
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.available()
diff --git a/user_scanner/user_scan/community/ubuntu_mate.py b/user_scanner/user_scan/community/ubuntu_mate.py
index 49b8e94..62275b6 100644
--- a/user_scanner/user_scan/community/ubuntu_mate.py
+++ b/user_scanner/user_scan/community/ubuntu_mate.py
@@ -12,6 +12,7 @@ def validate_ubuntu_mate(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_ubuntu_mate(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,10 +28,10 @@ def validate_ubuntu_mate(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://ubuntu-mate.community" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media)
+
return Result.error(f"Unexpected response status: {response.status_code}")
headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"}
diff --git a/user_scanner/user_scan/creator/boosty.py b/user_scanner/user_scan/creator/boosty.py
index 906ede6..070967e 100644
--- a/user_scanner/user_scan/creator/boosty.py
+++ b/user_scanner/user_scan/creator/boosty.py
@@ -15,6 +15,7 @@ def validate_boosty(user: str) -> Result:
# If it's a valid blog, it contains a unique ID or title
if "title" in data:
extra = {}
+ media = {}
if data.get("title"):
extra["display_name"] = data.get("title")
@@ -26,7 +27,7 @@ def validate_boosty(user: str) -> Result:
owner = data.get("owner", {})
if owner.get("avatarUrl"):
- extra["avatar_url"] = owner.get("avatarUrl")
+ media["avatar"] = owner.get("avatarUrl")
if owner.get("name") and owner.get("name") != data.get("title"):
extra["owner_name"] = owner.get("name")
@@ -38,7 +39,7 @@ def validate_boosty(user: str) -> Result:
if links:
extra["links"] = links
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
except Exception:
pass
diff --git a/user_scanner/user_scan/creator/substack.py b/user_scanner/user_scan/creator/substack.py
index e1eeacd..01459aa 100644
--- a/user_scanner/user_scan/creator/substack.py
+++ b/user_scanner/user_scan/creator/substack.py
@@ -15,6 +15,7 @@ def validate_substack(user: str) -> Result:
# Claimed profile has an id and name
if "id" in data:
extra = {}
+ media = {}
if data.get("name"):
extra["display_name"] = data.get("name")
if data.get("handle"):
@@ -22,7 +23,7 @@ def validate_substack(user: str) -> Result:
if data.get("bio"):
extra["bio"] = data.get("bio")
if data.get("photo_url"):
- extra["avatar_url"] = data.get("photo_url")
+ media["avatar"] = data.get("photo_url")
# Extract publication info if they have one
primary_pub = data.get("primaryPublication") or {}
@@ -37,7 +38,7 @@ def validate_substack(user: str) -> Result:
if data.get("followerCount") is not None:
extra["followers"] = int(data.get("followerCount"))
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
except Exception:
pass
diff --git a/user_scanner/user_scan/dev/coderwall.py b/user_scanner/user_scan/dev/coderwall.py
index e6c0b71..e1be18a 100644
--- a/user_scanner/user_scan/dev/coderwall.py
+++ b/user_scanner/user_scan/dev/coderwall.py
@@ -12,21 +12,22 @@ def validate_coderwall(user):
data = response.json()
if data and "username" in data:
extra = {}
+ media = {}
if data.get("id"): extra["id"] = data.get("id")
if data.get("name"): extra["name"] = data.get("name")
if data.get("location"): extra["location"] = data.get("location")
if data.get("karma") is not None: extra["karma"] = data.get("karma")
if data.get("company"): extra["company"] = data.get("company")
if data.get("about"): extra["about"] = data.get("about").strip()
- if data.get("thumbnail"): extra["avatar"] = data.get("thumbnail")
-
+ if data.get("thumbnail"): media["avatar"] = data.get("thumbnail")
+
accounts = data.get("accounts", {})
if accounts:
for platform, handle in accounts.items():
if handle:
extra[f"{platform}_handle"] = handle
-
- return Result.taken(extra=extra, url=show_url)
+
+ return Result.taken(extra=extra, media=media, url=show_url)
return Result.available(url=show_url)
elif response.status_code == 404:
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/dev/cratesio.py b/user_scanner/user_scan/dev/cratesio.py
index 7306733..18598a8 100644
--- a/user_scanner/user_scan/dev/cratesio.py
+++ b/user_scanner/user_scan/dev/cratesio.py
@@ -18,6 +18,7 @@ def validate_cratesio(user):
return Result.available()
elif response.status_code == 200:
extra = {}
+ media = {}
try:
data = response.json()
u_data = data.get("user", {})
@@ -27,14 +28,13 @@ def validate_cratesio(user):
if u_data.get("name"):
extra["name"] = u_data.get("name").strip()
if u_data.get("avatar"):
- extra["avatar"] = u_data.get("avatar")
+ media["avatar"] = u_data.get("avatar")
if u_data.get("url"):
extra["github_url"] = u_data.get("url")
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
else:
return Result.error(f"HTTP {response.status_code}")
return generic_validate(url, process, show_url=show_url, headers=headers)
-
diff --git a/user_scanner/user_scan/dev/elixir_forum.py b/user_scanner/user_scan/dev/elixir_forum.py
index 208024e..5e0ddd7 100644
--- a/user_scanner/user_scan/dev/elixir_forum.py
+++ b/user_scanner/user_scan/dev/elixir_forum.py
@@ -15,6 +15,7 @@ def validate_elixir_forum(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -22,7 +23,7 @@ def validate_elixir_forum(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -30,9 +31,9 @@ def validate_elixir_forum(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://forum.elixirforum.com" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra, url=show_url)
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media, url=show_url)
return Result.available(url=show_url)
elif response.status_code == 404:
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/dev/f_droid.py b/user_scanner/user_scan/dev/f_droid.py
index 4ab7b9c..01fc82d 100644
--- a/user_scanner/user_scan/dev/f_droid.py
+++ b/user_scanner/user_scan/dev/f_droid.py
@@ -12,6 +12,7 @@ def validate_f_droid(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_f_droid(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,10 +28,10 @@ def validate_f_droid(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://forum.f-droid.org" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media)
+
return Result.error(f"Unexpected response status: {response.status_code}")
headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"}
diff --git a/user_scanner/user_scan/dev/gitlab.py b/user_scanner/user_scan/dev/gitlab.py
index d340620..79d163f 100644
--- a/user_scanner/user_scan/dev/gitlab.py
+++ b/user_scanner/user_scan/dev/gitlab.py
@@ -21,6 +21,7 @@ def validate_gitlab(user):
else:
import re
extra = {}
+ media = {}
try:
u_data = data[0]
if u_data.get("id"):
@@ -31,21 +32,20 @@ def validate_gitlab(user):
extra["username"] = u_data.get("username").strip()
if u_data.get("state"):
extra["state"] = u_data.get("state")
-
+
avatar_url = u_data.get("avatar_url")
if avatar_url:
- extra["image"] = avatar_url
+ media["avatar"] = avatar_url
# Extract gravatar fields
m = re.search(r'gravatar\.com/avatar/([a-f0-9]{32})', avatar_url)
if m:
md5_hash = m.group(1)
- extra["gravatar_url"] = f"https://gravatar.com/{md5_hash}"
+ media["gravatar_url"] = f"https://gravatar.com/{md5_hash}"
extra["gravatar_username"] = u_data.get("username", "")
extra["gravatar_email_md5_hash"] = md5_hash
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error(f"Unexpected status or response: {response.status_code}")
return generic_validate(url, process, show_url=show_url, headers=headers)
-
diff --git a/user_scanner/user_scan/dev/hashicorp_discuss.py b/user_scanner/user_scan/dev/hashicorp_discuss.py
index 4ac887e..519588a 100644
--- a/user_scanner/user_scan/dev/hashicorp_discuss.py
+++ b/user_scanner/user_scan/dev/hashicorp_discuss.py
@@ -12,6 +12,7 @@ def validate_hashicorp_discuss(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_hashicorp_discuss(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,10 +28,10 @@ def validate_hashicorp_discuss(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://discuss.hashicorp.com" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media)
+
return Result.error(f"Unexpected response status: {response.status_code}")
headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"}
diff --git a/user_scanner/user_scan/dev/huggingface.py b/user_scanner/user_scan/dev/huggingface.py
index e3b511b..58ef78e 100644
--- a/user_scanner/user_scan/dev/huggingface.py
+++ b/user_scanner/user_scan/dev/huggingface.py
@@ -10,6 +10,7 @@ def validate_huggingface(user):
return Result.available()
elif response.status_code == 200:
extra = {}
+ media = {}
try:
data = response.json()
if data.get("fullname"):
@@ -28,12 +29,11 @@ def validate_huggingface(user):
avatar = data.get("avatarUrl")
if avatar.startswith("/"):
avatar = "https://huggingface.co" + avatar
- extra["avatar"] = avatar
+ media["avatar"] = avatar
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
else:
return Result.error(f"HTTP {response.status_code}")
return generic_validate(url, process, show_url=show_url, follow_redirects=True)
-
diff --git a/user_scanner/user_scan/dev/kotlin_discuss.py b/user_scanner/user_scan/dev/kotlin_discuss.py
index 78d907f..bedc158 100644
--- a/user_scanner/user_scan/dev/kotlin_discuss.py
+++ b/user_scanner/user_scan/dev/kotlin_discuss.py
@@ -12,6 +12,7 @@ def validate_kotlin_discuss(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_kotlin_discuss(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,10 +28,10 @@ def validate_kotlin_discuss(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://discuss.kotlinlang.org" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media)
+
return Result.error(f"Unexpected response status: {response.status_code}")
headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"}
diff --git a/user_scanner/user_scan/dev/leetcode.py b/user_scanner/user_scan/dev/leetcode.py
index d406ce3..288336e 100644
--- a/user_scanner/user_scan/dev/leetcode.py
+++ b/user_scanner/user_scan/dev/leetcode.py
@@ -16,7 +16,7 @@ def validate_leetcode(user: str) -> Result:
)
show_url = f"https://leetcode.com/u/{user}/"
-
+
query = '''query userPublicProfile($username: String!) { matchedUser(username: $username) { username profile { realName aboutMe userAvatar countryName company school ranking } } }'''
variables = json.dumps({"username": user})
url = f"https://leetcode.com/graphql?query={quote(query)}&variables={quote(variables)}"
@@ -32,22 +32,23 @@ def validate_leetcode(user: str) -> Result:
data = response.json()
if data.get("errors") and "That user does not exist" in str(data.get("errors")):
return Result.available(url=show_url)
-
+
matched_user = data.get("data", {}).get("matchedUser")
if matched_user:
extra = {}
+ media = {}
profile = matched_user.get("profile", {})
-
+
if profile.get("realName"): extra["fullname"] = profile.get("realName")
if profile.get("aboutMe"): extra["bio"] = profile.get("aboutMe")
- if profile.get("userAvatar"): extra["image"] = profile.get("userAvatar")
+ if profile.get("userAvatar"): media["image"] = profile.get("userAvatar")
if profile.get("countryName"): extra["country"] = profile.get("countryName")
if profile.get("company"): extra["company"] = profile.get("company")
if profile.get("school"): extra["school"] = profile.get("school")
if profile.get("ranking"): extra["ranking"] = profile.get("ranking")
-
- return Result.taken(extra=extra, url=show_url)
-
+
+ return Result.taken(extra=extra, media=media, url=show_url)
+
return Result.available(url=show_url)
else:
return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
diff --git a/user_scanner/user_scan/dev/python_discuss.py b/user_scanner/user_scan/dev/python_discuss.py
index 9da0c6f..83899de 100644
--- a/user_scanner/user_scan/dev/python_discuss.py
+++ b/user_scanner/user_scan/dev/python_discuss.py
@@ -12,6 +12,7 @@ def validate_python_discuss(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_python_discuss(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,10 +28,10 @@ def validate_python_discuss(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://discuss.python.org" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media)
+
return Result.error(f"Unexpected response status: {response.status_code}")
headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"}
diff --git a/user_scanner/user_scan/dev/rust_users.py b/user_scanner/user_scan/dev/rust_users.py
index b41c41b..6195f30 100644
--- a/user_scanner/user_scan/dev/rust_users.py
+++ b/user_scanner/user_scan/dev/rust_users.py
@@ -12,6 +12,7 @@ def validate_rust_users(user):
u = data.get("user", {})
if u:
extra = {}
+ media = {}
if u.get("id"): extra["id"] = u.get("id")
if u.get("name"): extra["name"] = u.get("name")
if u.get("username"): extra["username"] = u.get("username")
@@ -19,7 +20,7 @@ def validate_rust_users(user):
if u.get("last_posted_at"): extra["last_posted"] = u.get("last_posted_at")
if u.get("last_seen_at"): extra["last_seen"] = u.get("last_seen_at")
if u.get("created_at"): extra["registered"] = u.get("created_at")
-
+
# Resolve avatar
avatar = u.get("avatar_template")
if avatar:
@@ -27,10 +28,10 @@ def validate_rust_users(user):
avatar = avatar.format(size=120)
if avatar.startswith("/"):
avatar = "https://users.rust-lang.org" + avatar
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media)
+
return Result.error(f"Unexpected response status: {response.status_code}")
headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0"}
diff --git a/user_scanner/user_scan/dev/scratch.py b/user_scanner/user_scan/dev/scratch.py
index d2eda73..7321b53 100644
--- a/user_scanner/user_scan/dev/scratch.py
+++ b/user_scanner/user_scan/dev/scratch.py
@@ -14,26 +14,27 @@ def validate_scratch(user):
response = make_request(api_url, headers=headers, follow_redirects=True)
if response.status_code == 200:
data = response.json()
-
+
extra = {}
+ media = {}
if user_id := data.get("id"):
extra["id"] = str(user_id)
- if history := data.get("history", {}):
+ if history := data.get("history"):
if joined := history.get("joined"):
extra["joined"] = joined
if profile := data.get("profile", {}):
if images := profile.get("images", {}):
if avatar := images.get("90x90"):
- extra["avatar"] = avatar
+ media["avatar"] = avatar
if country := profile.get("country"):
extra["country"] = country
-
- return Result.taken(extra=extra, url=show_url)
-
+
+ return Result.taken(extra=extra, media=media, url=show_url)
+
elif response.status_code == 404:
return Result.available(url=show_url)
else:
return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
-
+
except Exception as e:
return Result.error(e, url=show_url)
diff --git a/user_scanner/user_scan/donation/buymeacoffee.py b/user_scanner/user_scan/donation/buymeacoffee.py
index 02ced70..32e3fd3 100644
--- a/user_scanner/user_scan/donation/buymeacoffee.py
+++ b/user_scanner/user_scan/donation/buymeacoffee.py
@@ -12,20 +12,21 @@ def validate_buymeacoffee(user):
response = make_request(url, headers=headers, follow_redirects=True)
if response.status_code == 200:
extra = {}
-
+ media = {}
+
# Extract Name (from og:title or title)
n = re.search(r'<meta property="og:title" content="([^"]+)"', response.text)
if not n: n = re.search(r'<title[^>]*>([^<]+?)(?:\s*-\s*Buymeacoffee)?</title>', response.text, re.IGNORECASE)
if n: extra['name'] = n.group(1).strip()
-
+
m = re.search(r'<meta name="description" content="([^"]+)"', response.text)
if not m: m = re.search(r'<meta property="og:description" content="([^"]+)"', response.text)
if m: extra['bio'] = m.group(1).strip()
-
+
img = re.search(r'<meta property="og:image" content="([^"]+)"', response.text)
- if img: extra['avatar'] = img.group(1)
-
- return Result.taken(extra=extra, url=show_url)
+ if img: media['avatar'] = img.group(1)
+
+ return Result.taken(extra=extra, media=media, url=show_url)
elif response.status_code == 404:
return Result.available(url=show_url)
else:
diff --git a/user_scanner/user_scan/donation/donatealerts.py b/user_scanner/user_scan/donation/donatealerts.py
index abf9936..f9b75be 100644
--- a/user_scanner/user_scan/donation/donatealerts.py
+++ b/user_scanner/user_scan/donation/donatealerts.py
@@ -7,14 +7,16 @@ def validate_donation_alerts(user):
def process(response):
if response.status_code == 200:
extra = {}
+ media = {}
+
try:
data = response.json().get("data", {})
if name := data.get("name"): extra["name"] = name
if currency := data.get("preferred_currency"): extra["currency"] = currency
- if avatar := data.get("avatar"): extra["avatar"] = avatar
+ if avatar := data.get("avatar"): media["avatar"] = avatar
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
elif response.status_code == 202 or (response.status_code == 200 and not response.json().get("success", True)):
return Result.available()
return Result.error(f"Unexpected status {response.status_code}")
diff --git a/user_scanner/user_scan/finance/advfn.py b/user_scanner/user_scan/finance/advfn.py
index 6b03de5..299a5c1 100644
--- a/user_scanner/user_scan/finance/advfn.py
+++ b/user_scanner/user_scan/finance/advfn.py
@@ -9,6 +9,7 @@ def validate_advfn(user):
def process(response):
if "Profile | ADVFN" in response.text:
extra = {}
+ media = {}
html = response.text
avatar_match = local_re.search(r'<img class="profile-pic" src="([^"]+)">', html)
@@ -16,7 +17,7 @@ def validate_advfn(user):
url_src = avatar_match.group(1)
if url_src.startswith("//"):
url_src = "https:" + url_src
- extra["avatar"] = url_src
+ media["avatar"] = url_src
joined_match = local_re.search(r'<span class="registration">Member since:\s*([^<]+)</span>', html)
if joined_match:
@@ -44,7 +45,7 @@ def validate_advfn(user):
if "has no comment yet" not in bio.lower():
extra["bio"] = bio
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if "ADVFN ERROR - Page Not Found" in response.text:
return Result.available()
@@ -52,4 +53,3 @@ def validate_advfn(user):
return Result.error("Unexpected response body, report it via GitHub issues.")
return generic_validate(url, process, show_url=show_url)
-
diff --git a/user_scanner/user_scan/finance/cropty.py b/user_scanner/user_scan/finance/cropty.py
index 55360da..d150bcf 100644
--- a/user_scanner/user_scan/finance/cropty.py
+++ b/user_scanner/user_scan/finance/cropty.py
@@ -25,13 +25,14 @@ def validate_cropty(user):
if isinstance(data_json, dict) and "data" in data_json:
data = data_json["data"]
extra = {}
+ media = {}
if name := data.get("name"): extra["name"] = name
if nickname := data.get("nickname"): extra["nickname"] = nickname
if image := data.get("image"):
if "defaults" not in image:
- extra["avatar"] = image
+ media["avatar"] = image
if ref_link := data.get("ref_link"): extra["referral_link"] = ref_link
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error("Unexpected 200 response shape (no 'data' or known error code)")
if response.status_code == 404:
@@ -39,4 +40,4 @@ def validate_cropty(user):
return Result.error(f"Unexpected status: {response.status_code}")
- return generic_validate(url, process, headers=headers, show_url=show_url)
\ No newline at end of file
+ return generic_validate(url, process, headers=headers, show_url=show_url)
diff --git a/user_scanner/user_scan/finance/destream.py b/user_scanner/user_scan/finance/destream.py
index 4e5d5b9..1d4e538 100644
--- a/user_scanner/user_scan/finance/destream.py
+++ b/user_scanner/user_scan/finance/destream.py
@@ -15,15 +15,16 @@ def validate_destream(user):
try:
data = response.json()
extra = {}
+ media = {}
if username := data.get("userName"):
extra["username"] = username
if avatar := data.get("logoImageUrl"):
- extra["avatar"] = avatar
+ media["avatar"] = avatar
if stream := data.get("liveStream"):
extra["live_stream"] = str(stream)
if channel := data.get("channel"):
extra["channel"] = str(channel)
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
except Exception:
pass
return Result.error("200 response status with no recognizable data, report it via GitHub issues")
diff --git a/user_scanner/user_scan/finance/etoro.py b/user_scanner/user_scan/finance/etoro.py
index f163619..08c4ff5 100644
--- a/user_scanner/user_scan/finance/etoro.py
+++ b/user_scanner/user_scan/finance/etoro.py
@@ -13,6 +13,7 @@ def validate_etoro(user):
def process(response):
if '"gcid":' in response.text:
extra = {}
+ media = {}
try:
data = response.json()
first = data.get("firstName")
@@ -33,13 +34,13 @@ def validate_etoro(user):
avatars = data.get("avatars")
if avatars and isinstance(avatars, list) and len(avatars) > 0:
- extra["avatar"] = avatars[0]["url"]
+ media["avatar"] = avatars[0]["url"]
if data.get("isVerified"):
extra["is_verified"] = True
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if '"ErrorCode":"NotFound"' in response.text:
return Result.available()
@@ -50,4 +51,3 @@ def validate_etoro(user):
return Result.error("Unexpected API response format.")
return generic_validate(url, process, headers=headers, show_url=show_url)
-
diff --git a/user_scanner/user_scan/finance/hamaha.py b/user_scanner/user_scan/finance/hamaha.py
index f6cceb6..ac438b9 100644
--- a/user_scanner/user_scan/finance/hamaha.py
+++ b/user_scanner/user_scan/finance/hamaha.py
@@ -13,6 +13,7 @@ def validate_hamaha(user):
def process(response):
if 'id="profile"' in response.text:
extra = {}
+ media = {}
try:
name_match = local_re.search(r'<h2>(.*?)</h2>', response.text)
if name_match:
@@ -24,7 +25,7 @@ def validate_hamaha(user):
if avatar_match:
avatar_val = avatar_match.group(1).strip()
if "_noavatar_user.gif" not in avatar_val:
- extra["avatar"] = avatar_val
+ media["avatar"] = avatar_val
bio_match = local_re.search(r'<h3>Обо мне</h3>.*?<td>(.*?)</td>', response.text, local_re.DOTALL)
if bio_match:
@@ -53,7 +54,7 @@ def validate_hamaha(user):
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if 'content="HAMAHA Биткоин форум' in response.text or 'быстрая регистрация' in response.text.lower():
return Result.available()
@@ -61,4 +62,3 @@ def validate_hamaha(user):
return Result.error("Unexpected response body.")
return generic_validate(url, process, headers=headers, show_url=show_url)
-
diff --git a/user_scanner/user_scan/gaming/apexlegends.py b/user_scanner/user_scan/gaming/apexlegends.py
index d97230b..edb45f6 100644
--- a/user_scanner/user_scan/gaming/apexlegends.py
+++ b/user_scanner/user_scan/gaming/apexlegends.py
@@ -17,26 +17,27 @@ def validate_apexlegends(user):
payload = response.json()
data = payload.get("data", {})
extra = {}
-
+ media = {}
+
p_info = data.get("platformInfo", {})
if handle := p_info.get("platformUserHandle"): extra["username"] = handle
- if avatar := p_info.get("avatarUrl"): extra["avatar"] = avatar
-
+ if avatar := p_info.get("avatarUrl"): media["avatar"] = avatar
+
u_info = data.get("userInfo", {})
if country := u_info.get("countryCode"): extra["country"] = country
if u_info.get("isPremium"): extra["premium"] = "Yes"
if u_info.get("isInfluencer"): extra["influencer"] = "Yes"
-
+
meta = data.get("metadata", {})
if legend := meta.get("activeLegendName"): extra["active_legend"] = legend
-
+
segments = data.get("segments", [])
if segments:
stats = segments[0].get("stats", {})
if lvl := stats.get("level"): extra["level"] = str(lvl.get("displayValue"))
if kills := stats.get("kills"): extra["kills"] = str(kills.get("displayValue"))
-
- return Result.taken(extra=extra)
+
+ return Result.taken(extra=extra, media=media)
except Exception:
pass
return Result.error("200 response status with no recognizable data, report it via GitHub issues")
diff --git a/user_scanner/user_scan/gaming/chess_com.py b/user_scanner/user_scan/gaming/chess_com.py
index cfbbc0f..676a1c9 100644
--- a/user_scanner/user_scan/gaming/chess_com.py
+++ b/user_scanner/user_scan/gaming/chess_com.py
@@ -30,6 +30,8 @@ def validate_chess_com(user: str) -> Result:
try:
data = response.json()
extra = {}
+ media = {}
+
if name := data.get("name"): extra["name"] = name
if username := data.get("username"): extra["username"] = username
if title := data.get("title"): extra["title"] = title
@@ -37,15 +39,15 @@ def validate_chess_com(user: str) -> Result:
if league := data.get("league"): extra["league"] = league
if location := data.get("location"): extra["location"] = location
if followers := data.get("followers"): extra["followers"] = str(followers)
- if avatar := data.get("avatar"): extra["avatar"] = avatar
+ if avatar := data.get("avatar"): media["avatar"] = avatar
if twitch := data.get("twitch_url"): extra["links"] = twitch
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
except Exception:
pass
return Result.error("200 response status with no recognizable data, report it via GitHub issues")
elif response.status_code == 404:
return Result.available()
-
+
return Result.error(f"Unexpected status code: {response.status_code}")
return generic_validate(url, process, show_url=show_url, headers=headers)
diff --git a/user_scanner/user_scan/gaming/nexusmods.py b/user_scanner/user_scan/gaming/nexusmods.py
index 2039c6e..ecd4a81 100644
--- a/user_scanner/user_scan/gaming/nexusmods.py
+++ b/user_scanner/user_scan/gaming/nexusmods.py
@@ -25,12 +25,13 @@ def validate_nexusmods(user: str) -> Result:
return Result.error("Profile response did not match the requested username")
extra = {}
+ media = {}
avatar = re.search(
r'<meta property="og:image" content="([^"]+)"', response.text
)
if avatar:
- extra["avatar"] = html.unescape(avatar.group(1))
- user_id = re.search(r"/(\d+)/100$", extra["avatar"])
+ media["avatar"] = html.unescape(avatar.group(1))
+ user_id = re.search(r"/(\d+)/100$", media["avatar"])
if user_id:
extra["user_id"] = int(user_id.group(1))
@@ -81,6 +82,6 @@ def validate_nexusmods(user: str) -> Result:
except (AttributeError, ValueError, json.JSONDecodeError):
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return impersonate_validate(url, process, show_url=url, allow_redirects=True)
diff --git a/user_scanner/user_scan/gaming/osu.py b/user_scanner/user_scan/gaming/osu.py
index 69cf677..6d624ec 100644
--- a/user_scanner/user_scan/gaming/osu.py
+++ b/user_scanner/user_scan/gaming/osu.py
@@ -13,6 +13,7 @@ def validate_osu(user):
def process(response):
if response.status_code == 200:
extra = {}
+ media = {}
title_match = local_re.search(r'<meta property="og:title" content="([^"]+)"', response.text)
if title_match:
title_val = title_match.group(1).replace("· player info", "").strip()
@@ -20,13 +21,13 @@ def validate_osu(user):
avatar_match = local_re.search(r'<meta property="og:image" content="([^"]+)"', response.text)
if avatar_match:
- extra["avatar"] = avatar_match.group(1).strip()
+ media["avatar"] = avatar_match.group(1).strip()
desc_match = local_re.search(r'<meta property="og:description" content="([^"]+)"', response.text)
if desc_match:
extra["rank"] = desc_match.group(1).strip()
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if response.status_code == 404:
return Result.available()
diff --git a/user_scanner/user_scan/gaming/roblox.py b/user_scanner/user_scan/gaming/roblox.py
index 52cfd79..cf26b66 100644
--- a/user_scanner/user_scan/gaming/roblox.py
+++ b/user_scanner/user_scan/gaming/roblox.py
@@ -80,14 +80,16 @@ def process(response):
"uid": uid,
"is verified": entry.get("hasVerifiedBadge"),
}
+ media = {}
if uid:
extra.update(_fetch_user_details(uid))
if get_avatar_url := _get_avatar_picture(uid):
- extra["avatar"] = get_avatar_url
+ media["avatar"] = get_avatar_url
return Result.taken(
extra=extra,
+ media=media,
url=f"https://www.roblox.com/users/{uid}" if uid else None,
)
diff --git a/user_scanner/user_scan/gaming/stackb.py b/user_scanner/user_scan/gaming/stackb.py
index d57a1f3..b0d7f81 100644
--- a/user_scanner/user_scan/gaming/stackb.py
+++ b/user_scanner/user_scan/gaming/stackb.py
@@ -87,6 +87,8 @@ def validate_stackb(user: str) -> Result:
return Result.error("Unexpected login check response", url=url)
extra = {}
+ media = {}
+
try:
profile_response = make_request(
url,
@@ -114,7 +116,7 @@ def validate_stackb(user: str) -> Result:
if name := profile.get("name"): extra["display_name"] = name
if description := profile.get("description"): extra["bio"] = description
- if image := profile.get("image"): extra["avatar"] = image
+ if image := profile.get("image"): media["avatar"] = image
if stats_text:
if rank_match := re.search(r"Ранг:\s*([^\.]+)", stats_text):
@@ -126,6 +128,6 @@ def validate_stackb(user: str) -> Result:
except Exception:
pass
- return Result.taken(extra=extra, url=url)
+ return Result.taken(extra=extra, media=media, url=url)
except Exception as exc:
return Result.error(exc, url=url)
diff --git a/user_scanner/user_scan/learning/pedsovet.py b/user_scanner/user_scan/learning/pedsovet.py
index 9c9aff9..b712714 100644
--- a/user_scanner/user_scan/learning/pedsovet.py
+++ b/user_scanner/user_scan/learning/pedsovet.py
@@ -39,6 +39,7 @@ def validate_pedsovet(user: str) -> Result:
and "Ссылка на профиль:" in response_text
):
extra = {"login": found_login}
+ media = {}
if id_match := re.search(r"/index/8-(\d+)", response_text):
extra["id"] = id_match.group(1)
@@ -63,7 +64,7 @@ def validate_pedsovet(user: str) -> Result:
avatar = "https:" + avatar
elif avatar.startswith("/"):
avatar = "https://pedsovet.su" + avatar
- extra["avatar"] = avatar
+ media["avatar"] = avatar
if about_match := re.search(
r'<div class="osebe">О себе</div>.*?<p class="clr">(.*?)</p>', response_text, re.IGNORECASE | re.DOTALL
@@ -73,7 +74,7 @@ def validate_pedsovet(user: str) -> Result:
if about != "Пользователь пока ничего не сообщил о себе.":
extra["about"] = about
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error("Unexpected response body")
diff --git a/user_scanner/user_scan/music/audiojungle.py b/user_scanner/user_scan/music/audiojungle.py
index 2f21c14..b2ba259 100644
--- a/user_scanner/user_scan/music/audiojungle.py
+++ b/user_scanner/user_scan/music/audiojungle.py
@@ -15,6 +15,7 @@ def validate_audiojungle(user):
return Result.available()
elif response.status_code == 200:
extra = {}
+ media = {}
try:
import re as local_re
# 1. Location
@@ -28,7 +29,7 @@ def validate_audiojungle(user):
# 3. Avatar
avatar_match = local_re.search(r'class="user-info-header__user-details">\s*<img[^>]*src="([^"]+)"', response.text, local_re.DOTALL)
if avatar_match:
- extra["avatar"] = avatar_match.group(1).strip()
+ media["avatar"] = avatar_match.group(1).strip()
# 4. Followers
followers_match = local_re.search(r'href="/user/[^/]+/followers">Followers\s*<span[^>]*>([0-9]+)</span>', response.text, local_re.DOTALL)
if followers_match:
@@ -39,9 +40,8 @@ def validate_audiojungle(user):
extra["following"] = int(following_match.group(1))
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
else:
return Result.error(f"HTTP {response.status_code}")
return generic_validate(url, process, show_url=show_url, headers=headers)
-
diff --git a/user_scanner/user_scan/music/bandcamp.py b/user_scanner/user_scan/music/bandcamp.py
index ed60a83..e0dbe2b 100644
--- a/user_scanner/user_scan/music/bandcamp.py
+++ b/user_scanner/user_scan/music/bandcamp.py
@@ -8,6 +8,8 @@ def validate_bandcamp(user):
def process(response):
if response.status_code == 200 and " collection | Bandcamp</title>" in response.text:
extra = {}
+ media = {}
+
try:
import re as local_re
import html as local_html
@@ -37,10 +39,10 @@ def validate_bandcamp(user):
extra["fav_genre"] = fan.get("fav_genre")
if fan.get("photo") and isinstance(fan["photo"], dict) and fan["photo"].get("image_id"):
img_id = fan["photo"]["image_id"]
- extra["avatar"] = f"https://f4.bcbits.com/img/00{img_id}_10.jpg"
+ media["avatar"] = f"https://f4.bcbits.com/img/00{img_id}_10.jpg"
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if response.status_code == 404 or "<h2>Sorry, that something isn’t here.</h2>" in response.text:
return Result.available()
@@ -48,4 +50,3 @@ def validate_bandcamp(user):
return Result.error("Unexpected response body, report it via GitHub issues.")
return generic_validate(url, process, show_url=show_url)
-
diff --git a/user_scanner/user_scan/music/bandlab.py b/user_scanner/user_scan/music/bandlab.py
index 74185c6..8172944 100644
--- a/user_scanner/user_scan/music/bandlab.py
+++ b/user_scanner/user_scan/music/bandlab.py
@@ -8,6 +8,7 @@ def validate_bandlab(user):
def process(response):
if response.status_code == 200 and "about" in response.text:
extra = {}
+ media = {}
try:
data = response.json()
if data.get("id"):
@@ -31,10 +32,10 @@ def validate_bandlab(user):
if counters.get("bands") is not None:
extra["bands"] = counters.get("bands")
if data.get("picture") and isinstance(data["picture"], dict) and data["picture"].get("url"):
- extra["avatar"] = data["picture"]["url"]
+ media["avatar"] = data["picture"]["url"]
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if response.status_code == 404 or "Couldn't find any matching element" in response.text:
return Result.available()
@@ -42,4 +43,3 @@ def validate_bandlab(user):
return Result.error("Unexpected response body, report it via GitHub issues.")
return generic_validate(url, process, show_url=show_url)
-
diff --git a/user_scanner/user_scan/music/beatstars.py b/user_scanner/user_scan/music/beatstars.py
index 836a142..406b370 100644
--- a/user_scanner/user_scan/music/beatstars.py
+++ b/user_scanner/user_scan/music/beatstars.py
@@ -43,6 +43,7 @@ def validate_beatstars(user):
return Result.available()
extra = {}
+ media = {}
try:
profile = identifier_data.get("profileDetails", {})
if profile:
@@ -50,11 +51,11 @@ def validate_beatstars(user):
if artwork:
avatar = artwork.get("fitInUrl") or artwork.get("url")
if avatar:
- extra["avatar"] = avatar
+ media["avatar"] = avatar
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
except (AttributeError, ValueError, KeyError):
return Result.error("Failed to decode server response, report it via GitHub issues")
diff --git a/user_scanner/user_scan/music/discogs.py b/user_scanner/user_scan/music/discogs.py
index 5632b02..f1644d2 100644
--- a/user_scanner/user_scan/music/discogs.py
+++ b/user_scanner/user_scan/music/discogs.py
@@ -8,6 +8,8 @@ def validate_discogs(user):
def process(response):
if response.status_code == 200 and "\"id\":" in response.text:
extra = {}
+ media = {}
+
try:
data = response.json()
if data.get("id"):
@@ -33,10 +35,10 @@ def validate_discogs(user):
if data.get("home_page"):
extra["website"] = data.get("home_page").strip()
if data.get("avatar_url"):
- extra["avatar"] = data.get("avatar_url")
+ media["avatar"] = data.get("avatar_url")
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if response.status_code == 404 or "\"message\": \"User does not exist or may have been deleted.\"" in response.text:
return Result.available()
@@ -44,4 +46,3 @@ def validate_discogs(user):
return Result.error("Unexpected response body, report it via GitHub issues.")
return generic_validate(url, process, show_url=show_url)
-
diff --git a/user_scanner/user_scan/music/lastfm.py b/user_scanner/user_scan/music/lastfm.py
index c2c863a..c45c0df 100644
--- a/user_scanner/user_scan/music/lastfm.py
+++ b/user_scanner/user_scan/music/lastfm.py
@@ -10,6 +10,7 @@ def validate_lastfm(user):
return Result.available()
elif response.status_code == 200:
extra = {}
+ media = {}
try:
import re as local_re
# 1. Display Name
@@ -31,10 +32,10 @@ def validate_lastfm(user):
# 5. Avatar
avatar_match = local_re.search(r'src="([^"]+)"[^>]*alt="Avatar for [^"]+"[^>]*itemprop="image"', response.text, local_re.DOTALL)
if avatar_match:
- extra["avatar"] = avatar_match.group(1).strip()
+ media["avatar"] = avatar_match.group(1).strip()
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
else:
return Result.error(f"HTTP {response.status_code}")
@@ -44,4 +45,3 @@ def validate_lastfm(user):
"Accept-Language": "en-US,en;q=0.9",
}
return generic_validate(url, process, show_url=show_url, headers=headers)
-
diff --git a/user_scanner/user_scan/music/mixcloud.py b/user_scanner/user_scan/music/mixcloud.py
index 1e2f5aa..00ce7c7 100644
--- a/user_scanner/user_scan/music/mixcloud.py
+++ b/user_scanner/user_scan/music/mixcloud.py
@@ -14,13 +14,14 @@ def validate_mixcloud(user):
response = make_request(api_url, headers=headers, follow_redirects=True)
if response.status_code == 200:
data = response.json()
-
- # If the user doesn't exist, mixcloud API typically returns 404,
+
+ # If the user doesn't exist, mixcloud API typically returns 404,
# but sometimes JSON contains an error key.
if "error" in data:
return Result.available(url=show_url)
extra = {}
+ media = {}
if name := data.get("name"):
extra["name"] = name
if follower_count := data.get("follower_count"):
@@ -30,14 +31,14 @@ def validate_mixcloud(user):
if pictures := data.get("pictures", {}):
# Try large, fallback to thumbnail or small
if avatar := pictures.get("large") or pictures.get("thumbnail") or pictures.get("small"):
- extra["avatar"] = avatar
-
- return Result.taken(extra=extra, url=show_url)
-
+ media["avatar"] = avatar
+
+ return Result.taken(extra=extra, media=media, url=show_url)
+
elif response.status_code == 404:
return Result.available(url=show_url)
else:
return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
-
+
except Exception as e:
return Result.error(e, url=show_url)
diff --git a/user_scanner/user_scan/music/soundcloud.py b/user_scanner/user_scan/music/soundcloud.py
index f078baa..e41fc37 100644
--- a/user_scanner/user_scan/music/soundcloud.py
+++ b/user_scanner/user_scan/music/soundcloud.py
@@ -26,6 +26,7 @@ def validate_soundcloud(user):
if is_taken:
extra = {}
+ media = {}
try:
import re as local_re
import json as local_json
@@ -55,12 +56,12 @@ def validate_soundcloud(user):
if u_data.get("likes_count") is not None:
extra["likes"] = u_data.get("likes_count")
if u_data.get("avatar_url"):
- extra["avatar"] = u_data.get("avatar_url")
+ media["avatar"] = u_data.get("avatar_url")
if u_data.get("verified"):
extra["verified"] = "Yes"
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error("Unexpected response, report it via GitHub issues")
diff --git a/user_scanner/user_scan/music/spotify.py b/user_scanner/user_scan/music/spotify.py
index 068be38..ed62a38 100644
--- a/user_scanner/user_scan/music/spotify.py
+++ b/user_scanner/user_scan/music/spotify.py
@@ -15,36 +15,37 @@ def validate_spotify(user):
if response.status_code == 200:
data = response.json()
item = data.get("item", {})
-
+
extra = {}
+ media = {}
if name := item.get("displayName"):
extra["name"] = name
if image := item.get("image"):
- extra["image"] = image
+ media["image"] = image
if created_at := item.get("createdAt"):
extra["created_at"] = created_at
if timezone := item.get("timezone"):
extra["timezone"] = timezone
-
+
# Extra profile metadata
profile = item.get("profile", {})
if bio := profile.get("bio"):
extra["bio"] = bio
if pronouns := profile.get("pronouns"):
extra["pronouns"] = pronouns
-
+
# Subscription info
if "isPro" in item:
extra["is_pro"] = str(item["isPro"])
if "isPlus" in item:
extra["is_plus"] = str(item["isPlus"])
- return Result.taken(extra=extra, url=show_url)
-
+ return Result.taken(extra=extra, media=media, url=show_url)
+
elif response.status_code == 404:
return Result.available(url=show_url)
else:
return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
-
+
except Exception as e:
return Result.error(e, url=show_url)
diff --git a/user_scanner/user_scan/other/calendly.py b/user_scanner/user_scan/other/calendly.py
index d3bf9a0..0f83ba1 100644
--- a/user_scanner/user_scan/other/calendly.py
+++ b/user_scanner/user_scan/other/calendly.py
@@ -17,23 +17,24 @@ def validate_calendly(user):
response = make_request(api_url, headers=headers, follow_redirects=True)
if response.status_code == 200:
data = response.json()
-
+
extra = {}
+ media = {}
if name := data.get("name"):
extra["name"] = name
if description := data.get("description"):
extra["description"] = description
if avatar := data.get("avatar_url"):
- extra["avatar"] = avatar
+ media["avatar"] = avatar
if org := data.get("organization_uuid"):
extra["organization_uuid"] = org
-
- return Result.taken(extra=extra, url=show_url)
-
+
+ return Result.taken(extra=extra, media=media, url=show_url)
+
elif response.status_code == 404:
return Result.available(url=show_url)
else:
return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
-
+
except Exception as e:
return Result.error(e, url=show_url)
diff --git a/user_scanner/user_scan/other/pastebin.py b/user_scanner/user_scan/other/pastebin.py
index d196eb0..d612be6 100644
--- a/user_scanner/user_scan/other/pastebin.py
+++ b/user_scanner/user_scan/other/pastebin.py
@@ -9,6 +9,8 @@ def validate_pastebin(user):
if response.status_code == 200:
if "info-bar" in response.text or "user-icon" in response.text:
extra = {}
+ media = {}
+
html = response.text
# Avatar
@@ -18,7 +20,7 @@ def validate_pastebin(user):
if "guest.png" not in avatar_src:
if avatar_src.startswith("/"):
avatar_src = "https://pastebin.com" + avatar_src
- extra["avatar"] = avatar_src
+ media["avatar"] = avatar_src
# Pro User
if 'class="pro"' in html:
@@ -54,7 +56,7 @@ def validate_pastebin(user):
if loc_match:
extra["location"] = loc_match.group(1).strip()
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.available()
@@ -64,4 +66,3 @@ def validate_pastebin(user):
return Result.error(f"Unexpected status code: {response.status_code}")
return generic_validate(url, process, show_url=show_url)
-
diff --git a/user_scanner/user_scan/other/polarsteps.py b/user_scanner/user_scan/other/polarsteps.py
index 80ce55a..a4a45e1 100644
--- a/user_scanner/user_scan/other/polarsteps.py
+++ b/user_scanner/user_scan/other/polarsteps.py
@@ -47,7 +47,6 @@ def validate_polarsteps(user: str) -> Result:
"last_name": data.get("last_name"),
"bio": data.get("description"),
"joined": data.get("creation_date"),
- "avatar": data.get("profile_image_path"),
"locality": location.get("locality"),
"administrative_area": location.get("administrative_area"),
"country": location.get("country"),
@@ -69,7 +68,8 @@ def validate_polarsteps(user: str) -> Result:
"furthest_country": stats.get("furthest_place_from_home_country"),
"furthest_distance_km": stats.get("furthest_place_from_home_km"),
}
- return Result.taken(extra=extra)
+ media = {"avatar": data.get("profile_image_path"),}
+ return Result.taken(extra=extra, media=media)
return Result.error(f"Unexpected response status: {response.status_code}")
diff --git a/user_scanner/user_scan/other/tripadvisor.py b/user_scanner/user_scan/other/tripadvisor.py
index 482c582..56295e7 100644
--- a/user_scanner/user_scan/other/tripadvisor.py
+++ b/user_scanner/user_scan/other/tripadvisor.py
@@ -18,14 +18,16 @@ def validate_tripadvisor(user):
def process(response):
if response.status_code == 200:
- return Result.taken(extra=_fetch_profile(user))
+ extra, media = _fetch_profile(user)
+ return Result.taken(extra=extra, media=media)
# Handles are case-canonicalized: a non-canonical casing 301-redirects
# to the real profile, so a redirect back to /Profile/ still means the
# account exists. Only a genuinely missing handle returns 404.
if response.status_code in (301, 302):
location = response.headers.get("location", "")
if "/profile/" in location.lower():
- return Result.taken(extra=_fetch_profile(user))
+ extra, media = _fetch_profile(user)
+ return Result.taken(extra=extra, media=media)
if response.status_code == 404:
return Result.available()
return Result.error(f"Unexpected status: {response.status_code}")
@@ -33,7 +35,7 @@ def validate_tripadvisor(user):
return impersonate_validate(url, process, warmup_url=WARMUP_URL, show_url=url)
-def _fetch_profile(user: str) -> dict:
+def _fetch_profile(user: str) -> tuple[dict, dict]:
payload = [
{
"variables": {"username": user},
@@ -48,25 +50,29 @@ def _fetch_profile(user: str) -> dict:
)
profiles = response.json()[0]["data"]["memberProfiles"]
profile = profiles[0] if profiles else None
- return _parse_profile(profile) if profile else {}
+ return _parse_profile(profile) if profile else ({}, {})
except Exception:
- return {}
+ return ({}, {})
-def _parse_profile(profile: dict) -> dict:
+def _parse_profile(profile: dict) -> tuple[dict, dict]:
sizes = (profile.get("avatar") or {}).get("photoSizes") or []
avatar = max(sizes, key=lambda p: p.get("width") or 0).get("url") if sizes else None
- return {
- "name": profile.get("displayName"),
- "user_id": profile.get("userId"),
- "bio": profile.get("bio"),
- "joined": profile.get("created"),
- "hometown": _extract_hometown(profile),
- "website": _decode_website(profile.get("website")),
- "verified": profile.get("isVerified"),
- "avatar": avatar,
- }
+ return (
+ {
+ "name": profile.get("displayName"),
+ "user_id": profile.get("userId"),
+ "bio": profile.get("bio"),
+ "joined": profile.get("created"),
+ "hometown": _extract_hometown(profile),
+ "website": _decode_website(profile.get("website")),
+ "verified": profile.get("isVerified"),
+ },
+ {
+ "avatar": avatar,
+ },
+ )
def _extract_hometown(profile: dict) -> str | None:
diff --git a/user_scanner/user_scan/other/vivino.py b/user_scanner/user_scan/other/vivino.py
index 4788813..8ed1f27 100644
--- a/user_scanner/user_scan/other/vivino.py
+++ b/user_scanner/user_scan/other/vivino.py
@@ -4,18 +4,19 @@ from user_scanner.core.orchestrator import Result, make_request
def validate_vivino(user):
api_url = f"https://api.vivino.com/users/{user}"
show_url = f"https://www.vivino.com/users/{user}"
-
+
headers = {
"User-Agent": get_random_user_agent(),
"Accept": "application/json",
}
-
+
resp = make_request(api_url, headers=headers, http2=True)
if resp.status_code == 200:
try:
data = resp.json()
if "id" in data:
extra = {}
+ media = {}
extra["id"] = data["id"]
if "alias" in data:
extra["alias"] = data["alias"]
@@ -27,10 +28,10 @@ def validate_vivino(user):
img = data["image"]["location"]
if img.startswith("//"):
img = "https:" + img
- extra["image"] = img
+ media["image"] = img
if "language" in data:
extra["language"] = data["language"]
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
except Exception:
pass
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/other/zomato.py b/user_scanner/user_scan/other/zomato.py
index e69c73c..9f82437 100644
--- a/user_scanner/user_scan/other/zomato.py
+++ b/user_scanner/user_scan/other/zomato.py
@@ -17,6 +17,8 @@ def validate_zomato(user):
match = local_re.search(r'window\.__PRELOADED_STATE__\s*=\s*([\s\S]*?);', response.text)
extra = {}
+ media = {}
+
if match:
try:
raw_json_str = match.group(1)
@@ -24,13 +26,13 @@ def validate_zomato(user):
decoded_js_str = json.loads(raw_json_str)
else:
decoded_js_str = json.loads(f'"{raw_json_str}"')
-
+
data = json.loads(decoded_js_str)
user_id = data.get("pages", {}).get("current", {}).get("userId")
if user_id:
user_id_str = str(user_id)
basic_info = data.get("pages", {}).get("user", {}).get(user_id_str, {}).get("sections", {}).get("SECTION_USER_BASIC_INFO", {})
-
+
if name := basic_info.get("name"):
extra["name"] = name.strip()
if bio := basic_info.get("bio"):
@@ -48,12 +50,12 @@ def validate_zomato(user):
if photos := basic_info.get("photoCount"):
extra["photos"] = int(photos)
if avatar := basic_info.get("profilePicture"):
- extra["avatar"] = avatar.strip()
+ media["avatar"] = avatar.strip()
if website := basic_info.get("websiteLink"):
extra["website"] = website.strip()
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if response.status_code == 404:
return Result.available()
diff --git a/user_scanner/user_scan/shopping/osta.py b/user_scanner/user_scan/shopping/osta.py
index 203cfb5..3d0b6a9 100644
--- a/user_scanner/user_scan/shopping/osta.py
+++ b/user_scanner/user_scan/shopping/osta.py
@@ -34,7 +34,9 @@ def validate_osta(user: str) -> Result:
about_url = f"{BASE_URL}/en?fuseaction=listing.aboutseller&user={user_id}"
if (about := _fetch(about_url, _extract_about)) is not None:
extra.update(about)
- return Result.taken(extra=extra, url=profile_url)
+
+ media = {"avatar": extra.pop("avatar")} if "avatar" in extra else {}
+ return Result.taken(extra=extra, media=media, url=profile_url)
return Result.error(f"Unexpected response status: {response.status_code}")
diff --git a/user_scanner/user_scan/social/35photo.py b/user_scanner/user_scan/social/35photo.py
index e3948eb..2021034 100644
--- a/user_scanner/user_scan/social/35photo.py
+++ b/user_scanner/user_scan/social/35photo.py
@@ -7,6 +7,7 @@ def validate_35photo(user):
def process(response):
if '<span title="Total photos' in response.text:
extra = {}
+ media = {}
try:
import re as local_re
# Name
@@ -27,15 +28,14 @@ def validate_35photo(user):
# Avatar
avatar_match = local_re.search(r'img class="avatar140"\s+src="([^"]+)"', response.text)
if avatar_match:
- extra["avatar"] = avatar_match.group(1)
+ media["avatar"] = avatar_match.group(1)
except Exception:
pass
- return Result.taken(extra=extra)
-
+ return Result.taken(extra=extra, media=media)
+
if "Catalogs of professional author" in response.text or response.status_code == 302:
return Result.available()
return Result.error("Unexpected response body, report it via GitHub issues.")
return generic_validate(url, process, show_url=show_url)
-
diff --git a/user_scanner/user_scan/social/anilist.py b/user_scanner/user_scan/social/anilist.py
index df3d6c4..fd34ad2 100644
--- a/user_scanner/user_scan/social/anilist.py
+++ b/user_scanner/user_scan/social/anilist.py
@@ -12,6 +12,7 @@ def validate_anilist(user):
def process(response):
if response.status_code == 200 and '"id":' in response.text:
extra = {}
+ media = {}
try:
data = response.json()
user_data = data.get("data", {}).get("User", {})
@@ -24,13 +25,13 @@ def validate_anilist(user):
clean_about = re.sub('<[^<]+?>', '', user_data.get("about"))
extra["about"] = clean_about.strip()
if user_data.get("avatar") and isinstance(user_data["avatar"], dict) and user_data["avatar"].get("large"):
- extra["avatar"] = user_data["avatar"]["large"]
+ media["avatar"] = user_data["avatar"]["large"]
if user_data.get("bannerImage"):
- extra["banner"] = user_data.get("bannerImage")
+ media["banner"] = user_data.get("bannerImage")
except Exception:
pass # Gracefully ignore any parsing issues and fallback to basic taken status
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if response.status_code == 404 or "Not Found" in response.text:
return Result.available()
@@ -40,4 +41,3 @@ def validate_anilist(user):
return generic_validate(
url, process, show_url=show_url, method="POST", json=payload, headers=headers
)
-
diff --git a/user_scanner/user_scan/social/bluesky.py b/user_scanner/user_scan/social/bluesky.py
index df6d1ef..d00a4f8 100644
--- a/user_scanner/user_scan/social/bluesky.py
+++ b/user_scanner/user_scan/social/bluesky.py
@@ -22,8 +22,10 @@ def validate_bluesky(user):
if data.get("postsCount") is not None:
extra["posts"] = data["postsCount"]
if data.get("avatar"):
- extra["avatar"] = data["avatar"]
- return Result.taken(extra=extra)
+ media = {"avatar": data["avatar"]}
+ else:
+ media = {}
+ return Result.taken(extra=extra, media=media)
if response.status_code == 400:
message = response.json().get("message")
diff --git a/user_scanner/user_scan/social/buzzfeed.py b/user_scanner/user_scan/social/buzzfeed.py
index da62217..bb779c0 100644
--- a/user_scanner/user_scan/social/buzzfeed.py
+++ b/user_scanner/user_scan/social/buzzfeed.py
@@ -15,6 +15,7 @@ def validate_buzzfeed(user: str) -> Result:
if r.status_code == 200:
extra = {}
+ media = {}
match = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', r.text)
if match:
try:
@@ -26,12 +27,12 @@ def validate_buzzfeed(user: str) -> Result:
extra["display_name"] = user_data.get("displayName")
if user_data.get("bio"):
extra["bio"] = user_data.get("bio")
-
+
if user_data.get("image"):
img = user_data.get("image")
if not img.startswith("http"):
img = f"https://img.buzzfeed.com/buzzfeed-static{img}"
- extra["avatar_url"] = img
+ media["avatar_url"] = img
if user_data.get("memberSince"):
try:
@@ -53,7 +54,7 @@ def validate_buzzfeed(user: str) -> Result:
if links:
extra["links"] = links
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
except Exception:
pass
diff --git a/user_scanner/user_scan/social/carrd.py b/user_scanner/user_scan/social/carrd.py
index b62f1fe..bb36eba 100644
--- a/user_scanner/user_scan/social/carrd.py
+++ b/user_scanner/user_scan/social/carrd.py
@@ -16,6 +16,7 @@ def validate_carrd(user: str) -> Result:
return Result.available()
extra = {}
+ media = {}
# Parse display name from h1
h1_match = re.search(r"<h1[^>]*>([^<]+)</h1>", r.text)
@@ -37,7 +38,7 @@ def validate_carrd(user: str) -> Result:
img_src = img_match.group(1).strip()
if img_src.startswith("assets/"):
img_src = f"https://{user}.carrd.co/{img_src}"
- extra["avatar_url"] = img_src
+ media["avatar_url"] = img_src
# Extract social links from anchor tags
links = []
@@ -47,7 +48,7 @@ def validate_carrd(user: str) -> Result:
if links:
extra["links"] = list(set(links))
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error(f"HTTP {r.status_code}")
diff --git a/user_scanner/user_scan/social/dailymotion.py b/user_scanner/user_scan/social/dailymotion.py
index 20e416f..e779474 100644
--- a/user_scanner/user_scan/social/dailymotion.py
+++ b/user_scanner/user_scan/social/dailymotion.py
@@ -14,7 +14,7 @@ def validate_dailymotion(user):
response = make_request(api_url, headers=headers, follow_redirects=True)
if response.status_code == 200:
data = response.json()
-
+
if "error" in data:
# Dailymotion API can return 200 with an error object
if data["error"].get("code") == 404:
@@ -22,12 +22,14 @@ def validate_dailymotion(user):
return Result.error(data["error"].get("message", "Unknown error"), url=show_url)
extra = {}
+ media = {}
+
if screenname := data.get("screenname"):
extra["screenname"] = screenname
if description := data.get("description"):
extra["description"] = description
if avatar := data.get("avatar_720_url"):
- extra["avatar"] = avatar
+ media["avatar"] = avatar
if followers := data.get("followers_total"):
extra["followers"] = str(followers)
if videos := data.get("videos_total"):
@@ -36,13 +38,13 @@ def validate_dailymotion(user):
extra["country"] = country
if verified := data.get("verified"):
extra["verified"] = str(verified)
-
- return Result.taken(extra=extra, url=show_url)
-
+
+ return Result.taken(extra=extra, media=media, url=show_url)
+
elif response.status_code == 404:
return Result.available(url=show_url)
else:
return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
-
+
except Exception as e:
return Result.error(e, url=show_url)
diff --git a/user_scanner/user_scan/social/flickr.py b/user_scanner/user_scan/social/flickr.py
index d24d568..2b07a64 100644
--- a/user_scanner/user_scan/social/flickr.py
+++ b/user_scanner/user_scan/social/flickr.py
@@ -15,6 +15,7 @@ def validate_flickr(user: str) -> Result:
if r.status_code == 200:
extra = {}
+ media = {}
match = re.search(r"modelExport:\s*(.*?),\s*auth", r.text)
if match:
try:
@@ -39,7 +40,7 @@ def validate_flickr(user: str) -> Result:
if avatar_retina:
if avatar_retina.startswith("//"):
avatar_retina = "https:" + avatar_retina
- extra["avatar_url"] = avatar_retina
+ media["avatar"] = avatar_retina
if profile.get("photoCount") is not None:
extra["photos"] = int(profile.get("photoCount"))
@@ -51,7 +52,7 @@ def validate_flickr(user: str) -> Result:
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error(f"HTTP {r.status_code}")
diff --git a/user_scanner/user_scan/social/giphy.py b/user_scanner/user_scan/social/giphy.py
index 1dcc754..07843df 100644
--- a/user_scanner/user_scan/social/giphy.py
+++ b/user_scanner/user_scan/social/giphy.py
@@ -21,6 +21,7 @@ def validate_giphy(user: str) -> Result:
or "giphy://shortcut/channel/" in r.text
):
extra = {}
+ media = {}
title_match = re.search(r"<title>([^<]+)</title>", r.text)
if title_match:
display_name = title_match.group(1).replace(" GIFs on GIPHY - Be Animated", "").strip()
@@ -36,9 +37,9 @@ def validate_giphy(user: str) -> Result:
r'<meta property="og:image" content="([^"]+)"', r.text
)
if image_match:
- extra["avatar_url"] = image_match.group(1).strip()
+ media["avatar_url"] = image_match.group(1).strip()
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error(f"HTTP {r.status_code}")
diff --git a/user_scanner/user_scan/social/gravatar.py b/user_scanner/user_scan/social/gravatar.py
index c5f61c4..29a2f99 100644
--- a/user_scanner/user_scan/social/gravatar.py
+++ b/user_scanner/user_scan/social/gravatar.py
@@ -21,13 +21,14 @@ def validate_gravatar(user: str) -> Result:
if response.status_code == 200:
data = response.json()
extra = {}
+ media = {}
if "entry" in data and len(data["entry"]) > 0:
entry = data["entry"][0]
if entry.get("hash"):
extra["gravatar_id"] = entry["hash"]
if entry.get("thumbnailUrl"):
- extra["image"] = entry["thumbnailUrl"]
+ media["avatar"] = entry["thumbnailUrl"]
if entry.get("preferredUsername"):
extra["username"] = entry["preferredUsername"]
if entry.get("name", {}).get("formatted"):
@@ -48,7 +49,7 @@ def validate_gravatar(user: str) -> Result:
if links:
extra["links"] = ", ".join(links)
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
return Result.error(f"Unexpected status: {response.status_code}", url=show_url)
except Exception as e:
diff --git a/user_scanner/user_scan/social/habr.py b/user_scanner/user_scan/social/habr.py
index c7661a9..5541435 100644
--- a/user_scanner/user_scan/social/habr.py
+++ b/user_scanner/user_scan/social/habr.py
@@ -19,6 +19,7 @@ def validate_habr(user):
if user_key:
user_info = data[user_key]
extra = {}
+ media = {}
if user_info.get('fullname'): extra['name'] = user_info.get('fullname')
if user_info.get('speciality'): extra['speciality'] = user_info.get('speciality')
if user_info.get('rating') is not None: extra['rating'] = user_info.get('rating')
@@ -28,8 +29,8 @@ def validate_habr(user):
if user_info.get('counterStats', {}).get('postCount') is not None: extra['posts'] = user_info.get('counterStats', {}).get('postCount')
if user_info.get('counterStats', {}).get('commentCount') is not None: extra['comments'] = user_info.get('counterStats', {}).get('commentCount')
if user_info.get('registerDateTime'): extra['registered'] = user_info.get('registerDateTime')
- if user_info.get('avatarUrl'): extra['avatar'] = user_info.get('avatarUrl')
- return Result.taken(extra=extra)
+ if user_info.get('avatarUrl'): media['avatar'] = user_info.get('avatarUrl')
+ return Result.taken(extra=extra, media=media)
except Exception:
pass
return Result.error(f"Unexpected response status: {response.status_code}")
diff --git a/user_scanner/user_scan/social/instagram.py b/user_scanner/user_scan/social/instagram.py
index 38fe850..75d8795 100644
--- a/user_scanner/user_scan/social/instagram.py
+++ b/user_scanner/user_scan/social/instagram.py
@@ -35,6 +35,7 @@ def validate_instagram(user: str) -> Result:
user_data = data.get("data", {}).get("user")
if user_data:
extra = {}
+ media = {}
# Base Fields
if user_data.get("username"):
@@ -44,7 +45,7 @@ def validate_instagram(user: str) -> Result:
if user_data.get("id"):
extra["id"] = user_data.get("id")
if user_data.get("profile_pic_url_hd"):
- extra["image"] = user_data.get("profile_pic_url_hd")
+ media["image"] = user_data.get("profile_pic_url_hd")
if user_data.get("biography"):
extra["bio"] = user_data.get("biography")
@@ -76,7 +77,7 @@ def validate_instagram(user: str) -> Result:
extra["following_count"] = user_data.get(
"edge_follow", {}).get("count")
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
else:
# If API response succeeded but user object is empty/null, the user does not exist
return Result.available(url=show_url)
diff --git a/user_scanner/user_scan/social/mastodon.py b/user_scanner/user_scan/social/mastodon.py
index 5d93ee7..39bd8b6 100644
--- a/user_scanner/user_scan/social/mastodon.py
+++ b/user_scanner/user_scan/social/mastodon.py
@@ -24,6 +24,7 @@ def validate_mastodon(user: str) -> Result:
return Result.available()
elif response.status_code == 200:
extra = {}
+ media = {}
try:
data = response.json()
if data.get("id"):
@@ -41,14 +42,13 @@ def validate_mastodon(user: str) -> Result:
if data.get("statuses_count") is not None:
extra["posts"] = data.get("statuses_count")
if data.get("avatar"):
- extra["avatar"] = data.get("avatar")
+ media["avatar"] = data.get("avatar")
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
else:
return Result.error(f"HTTP {response.status_code}")
return generic_validate(
url, process, show_url=show_url, follow_redirects=True
)
-
diff --git a/user_scanner/user_scan/social/openstreetmap.py b/user_scanner/user_scan/social/openstreetmap.py
index e7bd9f8..bb0915d 100644
--- a/user_scanner/user_scan/social/openstreetmap.py
+++ b/user_scanner/user_scan/social/openstreetmap.py
@@ -14,6 +14,7 @@ def validate_openstreetmap(user):
return Result.available()
if "Mapper since" in response.text:
extra = {}
+ media = {}
try:
import re as local_re
match = local_re.search(r"Mapper since:</dt>\s*<dd[^>]*>([^<]+)</dd>", response.text)
@@ -24,10 +25,10 @@ def validate_openstreetmap(user):
img_url = avatar_match.group(1)
if img_url.startswith("/"):
img_url = "https://www.openstreetmap.org" + img_url
- extra["avatar"] = img_url
+ media["avatar"] = img_url
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
if "does not exist" in response.text:
return Result.available()
return Result.error()
diff --git a/user_scanner/user_scan/social/pinterest.py b/user_scanner/user_scan/social/pinterest.py
index 163c00c..38baa20 100644
--- a/user_scanner/user_scan/social/pinterest.py
+++ b/user_scanner/user_scan/social/pinterest.py
@@ -10,8 +10,9 @@ def validate_pinterest(user):
if response.status_code == 200:
if "User not found." in response.text:
return Result.available()
-
+
extra = {}
+ media = {}
try:
import re as local_re
import json as local_json
@@ -38,15 +39,14 @@ def validate_pinterest(user):
if u_data.get("website_url"):
extra["website"] = u_data.get("website_url")
if u_data.get("image_xlarge_url"):
- extra["avatar"] = u_data.get("image_xlarge_url")
+ media["avatar"] = u_data.get("image_xlarge_url")
elif u_data.get("image_medium_url"):
- extra["avatar"] = u_data.get("image_medium_url")
+ media["avatar"] = u_data.get("image_medium_url")
except Exception:
pass
-
- return Result.taken(extra=extra)
+
+ return Result.taken(extra=extra, media=media)
else:
return Result.error("Invalid status code")
return generic_validate(url, process, show_url=show_url, follow_redirects=True)
-
diff --git a/user_scanner/user_scan/social/snapchat.py b/user_scanner/user_scan/social/snapchat.py
index 82cb4b4..00c8e45 100644
--- a/user_scanner/user_scan/social/snapchat.py
+++ b/user_scanner/user_scan/social/snapchat.py
@@ -27,6 +27,7 @@ def validate_snapchat(user):
return Result.available()
elif response.status_code == 200:
extra = {}
+ media = {}
try:
import re as local_re
import json as local_json
@@ -39,14 +40,13 @@ def validate_snapchat(user):
if u_info.get("displayName"):
extra["display_name"] = u_info.get("displayName")
if u_info.get("snapcodeImageUrl"):
- extra["snapcode"] = u_info.get("snapcodeImageUrl")
+ media["snapcode"] = u_info.get("snapcodeImageUrl")
except Exception:
pass
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
else:
return Result.error(f"HTTP {response.status_code}")
return generic_validate(
url, process, show_url=show_url, headers=headers, follow_redirects=True
)
-
diff --git a/user_scanner/user_scan/social/speakerdeck.py b/user_scanner/user_scan/social/speakerdeck.py
index cf19b93..0f8d9f8 100644
--- a/user_scanner/user_scan/social/speakerdeck.py
+++ b/user_scanner/user_scan/social/speakerdeck.py
@@ -17,6 +17,7 @@ def validate_speakerdeck(user: str) -> Result:
return Result.available()
extra = {}
+ media = {}
# Extract display name
name_match = re.search(r"<h1>\s*([^\n<]+)\s*</h1>", r.text)
if name_match:
@@ -25,7 +26,7 @@ def validate_speakerdeck(user: str) -> Result:
# Extract avatar URL
avatar_match = re.search(r'<img[^>]+class="[^"]*avatar[^"]*"[^>]+src="([^"]+)"', r.text)
if avatar_match:
- extra["avatar_url"] = avatar_match.group(1).strip()
+ media["avatar_url"] = avatar_match.group(1).strip()
# Extract decks count
decks_match = re.search(r"([0-9]+)\s*Decks", r.text)
@@ -42,7 +43,7 @@ def validate_speakerdeck(user: str) -> Result:
if followers_match:
extra["followers"] = int(followers_match.group(1))
- return Result.taken(extra=extra)
+ return Result.taken(extra=extra, media=media)
return Result.error(f"HTTP {r.status_code}")
diff --git a/user_scanner/user_scan/social/tumblr.py b/user_scanner/user_scan/social/tumblr.py
index ef6099b..5eaa4a8 100644
--- a/user_scanner/user_scan/social/tumblr.py
+++ b/user_scanner/user_scan/social/tumblr.py
@@ -11,21 +11,22 @@ def validate_tumblr(user):
response = make_request(url, headers=headers)
if response.status_code == 200:
extra = {}
+ media = {}
title = re.search(r'<title[^>]*>(.*?)</title>', response.text, re.IGNORECASE)
if title:
extra["title"] = title.group(1).strip()
-
+
desc = re.search(r'<meta[^>]*name="description"[^>]*content="([^"]+)"', response.text, re.IGNORECASE)
if not desc:
desc = re.search(r'<meta[^>]*property="og:description"[^>]*content="([^"]+)"', response.text, re.IGNORECASE)
if desc:
extra["description"] = desc.group(1).strip()
-
+
img = re.search(r'<meta[^>]*property="og:image"[^>]*content="([^"]+)"', response.text, re.IGNORECASE)
if img:
- extra["avatar"] = img.group(1).strip()
-
- return Result.taken(extra=extra, url=show_url)
+ media["avatar"] = img.group(1).strip()
+
+ return Result.taken(extra=extra, media=media, url=show_url)
elif response.status_code == 404:
return Result.available(url=show_url)
else:
diff --git a/user_scanner/user_scan/social/x.py b/user_scanner/user_scan/social/x.py
index 5da0cf6..3e4b542 100644
--- a/user_scanner/user_scan/social/x.py
+++ b/user_scanner/user_scan/social/x.py
@@ -10,23 +10,24 @@ def validate_x(user):
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "application/json",
}
-
+
try:
vx_response = make_request(vx_url, headers=vx_headers, follow_redirects=True)
if vx_response.status_code == 200:
data = vx_response.json()
extra = {}
+ media = {}
if name := data.get("name"): extra["name"] = name
if bio := data.get("description"): extra["bio"] = bio
if loc := data.get("location"): extra["location"] = loc
if created := data.get("created_at"): extra["created_at"] = created
if followers := data.get("followers_count"): extra["followers"] = str(followers)
if following := data.get("following_count"): extra["following"] = str(following)
- if avatar := data.get("profile_image_url"): extra["avatar"] = avatar.replace("_normal", "")
+ if avatar := data.get("profile_image_url"): media["avatar"] = avatar.replace("_normal", "")
if protected := data.get("protected"): extra["protected"] = str(protected)
if tweets := data.get("tweet_count"): extra["tweets"] = str(tweets)
-
- return Result.taken(extra=extra, url=show_url)
+
+ return Result.taken(extra=extra, media=media, url=show_url)
except Exception:
pass
diff --git a/user_scanner/user_scan/social/youtube.py b/user_scanner/user_scan/social/youtube.py
index d961c1e..5ccb0be 100644
--- a/user_scanner/user_scan/social/youtube.py
+++ b/user_scanner/user_scan/social/youtube.py
@@ -21,6 +21,7 @@ def validate_youtube(user) -> Result:
return Result.available(url=show_url)
extra = {}
+ media = {}
marker = "var ytInitialData = "
start = response.text.find(marker)
if start == -1:
@@ -41,7 +42,7 @@ def validate_youtube(user) -> Result:
if keywords := meta.get("keywords"):
extra["keywords"] = keywords
if thumbs := meta.get("avatar", {}).get("thumbnails"):
- extra["image"] = thumbs[0].get("url")
+ media["image"] = thumbs[0].get("url")
except json.JSONDecodeError:
return Result.error("Could not confirm YouTube channel", url=show_url)
@@ -49,7 +50,7 @@ def validate_youtube(user) -> Result:
if subs:
extra["subscribers"] = subs.group(1)
- return Result.taken(extra=extra, url=show_url)
+ return Result.taken(extra=extra, media=media, url=show_url)
elif response.status_code == 404:
return Result.available(url=show_url)
else: