Skip to content

🕵️‍♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.

fix(core): group multi-module scans by category and prevent header bleeding (#641)

Kaif687dff5c40f9
diff --git a/tests/test_email_orchestrator.py b/tests/test_email_orchestrator.py
index b1d8db1..4f65b1e 100644
--- a/tests/test_email_orchestrator.py
+++ b/tests/test_email_orchestrator.py
@@ -8,3 +8,60 @@ def test_set_concurrency():
# restore
set_concurrency(original_max)
+
+
+def test_run_email_batch_multiple_categories_grouped(capsys, monkeypatch):
+ import types
+ from user_scanner.core import email_orchestrator
+ from user_scanner.core.helpers import ScanConfig
+ from user_scanner.core.result import Result
+
+ mod1 = types.ModuleType("cat_a.site1")
+ mod1.__file__ = "/path/to/email_scan/cat_a/site1.py"
+
+ async def val1(e):
+ return Result.taken(username=e)
+
+ setattr(mod1, "validate_site1", val1)
+
+ mod2 = types.ModuleType("cat_b.site2")
+ mod2.__file__ = "/path/to/email_scan/cat_b/site2.py"
+
+ async def val2(e):
+ return Result.taken(username=e)
+
+ setattr(mod2, "validate_site2", val2)
+
+ mod3 = types.ModuleType("cat_a.site3")
+ mod3.__file__ = "/path/to/email_scan/cat_a/site3.py"
+
+ async def val3(e):
+ return Result.taken(username=e)
+
+ setattr(mod3, "validate_site3", val3)
+
+ monkeypatch.setattr(email_orchestrator, "find_category", lambda m: m.__name__.split(".")[0].capitalize())
+ monkeypatch.setattr(email_orchestrator, "get_site_name", lambda m: m.__name__.split(".")[-1].capitalize())
+
+ results = email_orchestrator.run_email_module_batch([mod1, mod2, mod3], "[email protected]", ScanConfig(show_all=True))
+ assert len(results) == 3
+
+ out = capsys.readouterr().out
+ assert "== CAT_A SITES ==" in out
+ assert "== CAT_B SITES ==" in out
+
+ cat_a_pos = out.find("== CAT_A SITES ==")
+ cat_b_pos = out.find("== CAT_B SITES ==")
+ assert cat_a_pos != -1 and cat_b_pos != -1
+ assert cat_a_pos < cat_b_pos
+
+ site1_pos = out.find("Site1")
+ site2_pos = out.find("Site2")
+ site3_pos = out.find("Site3")
+
+ # Site1 and Site3 belong to Cat_a and must appear BEFORE Cat_b header
+ assert cat_a_pos < site1_pos < cat_b_pos
+ assert cat_a_pos < site3_pos < cat_b_pos
+ # Site2 belongs to Cat_b and must appear AFTER Cat_b header
+ assert site2_pos > cat_b_pos
+
diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py
index 588b0bc..3f8efec 100644
--- a/tests/test_orchestrator.py
+++ b/tests/test_orchestrator.py
@@ -62,3 +62,43 @@ def test_set_concurrency():
# restore
set_concurrency(original_max)
+
+
+def test_run_batch_multiple_categories_grouped(capsys, monkeypatch):
+ mod1 = types.ModuleType("cat_a.site1")
+ mod1.__file__ = "/path/to/user_scan/cat_a/site1.py"
+ setattr(mod1, "validate_site1", lambda u: Result.taken(username=u))
+
+ mod2 = types.ModuleType("cat_b.site2")
+ mod2.__file__ = "/path/to/user_scan/cat_b/site2.py"
+ setattr(mod2, "validate_site2", lambda u: Result.taken(username=u))
+
+ mod3 = types.ModuleType("cat_a.site3")
+ mod3.__file__ = "/path/to/user_scan/cat_a/site3.py"
+ setattr(mod3, "validate_site3", lambda u: Result.taken(username=u))
+
+ monkeypatch.setattr(orchestrator, "find_category", lambda m: m.__name__.split(".")[0].capitalize())
+ monkeypatch.setattr(orchestrator, "get_site_name", lambda m: m.__name__.split(".")[-1].capitalize())
+
+ results = orchestrator.run_user_module([mod1, mod2, mod3], "alice", ScanConfig(show_all=True))
+ assert len(results) == 3
+
+ out = capsys.readouterr().out
+ assert "== CAT_A SITES ==" in out
+ assert "== CAT_B SITES ==" in out
+
+ cat_a_pos = out.find("== CAT_A SITES ==")
+ cat_b_pos = out.find("== CAT_B SITES ==")
+ assert cat_a_pos != -1 and cat_b_pos != -1
+ assert cat_a_pos < cat_b_pos
+
+ site1_pos = out.find("Site1")
+ site2_pos = out.find("Site2")
+ site3_pos = out.find("Site3")
+
+ # Site1 and Site3 belong to Cat_a and must appear BEFORE Cat_b header
+ assert cat_a_pos < site1_pos < cat_b_pos
+ assert cat_a_pos < site3_pos < cat_b_pos
+ # Site2 belongs to Cat_b and must appear AFTER Cat_b header
+ assert site2_pos > cat_b_pos
+
diff --git a/user_scanner/core/email_orchestrator.py b/user_scanner/core/email_orchestrator.py
index 6fa4403..9c16edd 100644
--- a/user_scanner/core/email_orchestrator.py
+++ b/user_scanner/core/email_orchestrator.py
@@ -2,7 +2,7 @@ import asyncio
import httpx
from pathlib import Path
from types import ModuleType
-from typing import List, Optional, Set, Union, Callable
+from typing import Callable, Dict, List, Optional, Set, Union
from colorama import Fore, Style
@@ -65,6 +65,7 @@ async def _async_worker(
sem: asyncio.Semaphore,
configs: ScanConfig,
printed_cats: Optional[Set] = None,
+ cat_override: Optional[str] = None,
on_start: Optional[Callable[[str], None]] = None,
) -> Result:
async with sem:
@@ -72,7 +73,7 @@ async def _async_worker(
if on_start:
on_start(site_name)
func = get_scan_func(module)
- actual_cat = find_category(module) or "Email"
+ actual_cat = cat_override or find_category(module) or "Email"
params = {
"site_name": site_name.capitalize(),
@@ -109,13 +110,26 @@ async def _run_batch(
email: str,
configs: ScanConfig,
printed_cats: Optional[Set] = None,
+ cat_override: Optional[str] = None,
+ sem: Optional[asyncio.Semaphore] = None,
) -> List[Result]:
if not modules:
return []
- sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+ if sem is None:
+ sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+
+ if printed_cats is None:
+ printed_cats = set()
+
results = []
-
+
+ category_map: Dict[str, List[ModuleType]] = {}
+ for module in modules:
+ cat = cat_override or find_category(module) or "Email"
+ display_cat = cat.capitalize()
+ category_map.setdefault(display_cat, []).append(module)
+
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
@@ -129,33 +143,49 @@ async def _run_batch(
def on_start_cb(site: str):
progress.update(task_id, description=f"[cyan]Scanning {email}... ({site})")
- tasks = []
- for module in modules:
- t = asyncio.create_task(
- _async_worker(
- module,
- email,
- sem,
- configs,
- printed_cats=printed_cats,
- on_start=on_start_cb,
+ # 1. Pre-spawn all tasks for all categories (global concurrency)
+ spawned_category_tasks = []
+ for cat_name, cat_modules in category_map.items():
+ tasks = []
+ for module in cat_modules:
+ t = asyncio.create_task(
+ _async_worker(
+ module,
+ email,
+ sem,
+ configs,
+ cat_override=cat_name,
+ on_start=on_start_cb,
+ )
)
- )
- t.add_done_callback(lambda t: progress.advance(task_id))
- tasks.append(t)
-
- for coro in asyncio.as_completed(tasks):
- result = await coro
- actual_cat = result.category or "Unknown"
- if configs.show_all or result.is_visible(configs):
- if printed_cats is not None and actual_cat not in printed_cats:
+ t.add_done_callback(lambda t: progress.advance(task_id))
+ tasks.append(t)
+ spawned_category_tasks.append((cat_name, tasks))
+
+ # 2. Await tasks category by category to stream grouped output
+ for cat_name, tasks in spawned_category_tasks:
+ if not tasks:
+ continue
+
+ if configs.show_all:
+ if cat_name not in printed_cats:
print(
- f"\n{Fore.MAGENTA}== {actual_cat.upper()} SITES =={Style.RESET_ALL}"
+ f"\n{Fore.MAGENTA}== {cat_name.upper()} SITES =={Style.RESET_ALL}"
)
- printed_cats.add(actual_cat)
+ printed_cats.add(cat_name)
- result.show(configs)
- results.append(result)
+ for coro in asyncio.as_completed(tasks):
+ result = await coro
+ if configs.show_all or result.is_visible(configs):
+ cat = result.category or cat_name
+ if cat not in printed_cats:
+ print(
+ f"\n{Fore.MAGENTA}== {cat.upper()} SITES =={Style.RESET_ALL}"
+ )
+ printed_cats.add(cat)
+
+ result.show(configs)
+ results.append(result)
return results
@@ -195,6 +225,7 @@ async def _run_email_category_batch_async(
email,
configs,
printed_cats=printed_cats,
+ cat_override=cat_name,
)
def run_email_category_batch(
@@ -246,7 +277,7 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[R
email,
sem,
configs,
- printed_cats=printed_cats,
+ cat_override=display_name,
on_start=on_start_cb,
)
)
@@ -259,18 +290,19 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[R
continue
if configs.show_all:
- print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")
- printed_cats.add(display_name)
+ if display_name not in printed_cats:
+ print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")
+ printed_cats.add(display_name)
for coro in asyncio.as_completed(tasks):
result = await coro
if configs.show_all or result.is_visible(configs):
- display_name = result.category or "Unknown"
- if display_name not in printed_cats:
+ cat = result.category or display_name
+ if cat not in printed_cats:
print(
- f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}"
+ f"\n{Fore.MAGENTA}== {cat.upper()} SITES =={Style.RESET_ALL}"
)
- printed_cats.add(display_name)
+ printed_cats.add(cat)
result.show(configs)
all_results.append(result)
diff --git a/user_scanner/core/orchestrator.py b/user_scanner/core/orchestrator.py
index 6d511a7..c481b80 100644
--- a/user_scanner/core/orchestrator.py
+++ b/user_scanner/core/orchestrator.py
@@ -84,13 +84,25 @@ async def _run_batch(
configs: ScanConfig,
printed_cats: Optional[Set] = None,
cat_override: Optional[str] = None,
- sem: Optional[asyncio.Semaphore] = None
+ sem: Optional[asyncio.Semaphore] = None,
) -> List[Result]:
+ if not modules:
+ return []
+
if sem is None:
sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
-
+
+ if printed_cats is None:
+ printed_cats = set()
+
results = []
-
+
+ category_map: Dict[str, List[ModuleType]] = {}
+ for module in modules:
+ cat = cat_override or find_category(module) or "Unknown"
+ display_cat = cat.capitalize()
+ category_map.setdefault(display_cat, []).append(module)
+
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
@@ -104,27 +116,47 @@ async def _run_batch(
def on_start_cb(site: str):
progress.update(task_id, description=f"[cyan]Scanning {username}... ({site})")
- tasks = []
- for module in modules:
- t = asyncio.create_task(
- _async_worker(module, username, sem, configs, cat_override=cat_override, on_start=on_start_cb)
- )
- t.add_done_callback(lambda t: progress.advance(task_id))
- tasks.append(t)
+ # 1. Pre-spawn all tasks for all categories (global concurrency)
+ spawned_category_tasks = []
+ for cat_name, cat_modules in category_map.items():
+ tasks = []
+ for module in cat_modules:
+ t = asyncio.create_task(
+ _async_worker(
+ module,
+ username,
+ sem,
+ configs,
+ cat_override=cat_name,
+ on_start=on_start_cb,
+ )
+ )
+ t.add_done_callback(lambda t: progress.advance(task_id))
+ tasks.append(t)
+ spawned_category_tasks.append((cat_name, tasks))
+
+ # 2. Await tasks category by category to stream grouped output
+ for cat_name, tasks in spawned_category_tasks:
+ if not tasks:
+ continue
+
+ if configs.show_all:
+ if cat_name not in printed_cats:
+ print(f"\n{Fore.MAGENTA}== {cat_name.upper()} SITES =={Style.RESET_ALL}")
+ printed_cats.add(cat_name)
+
+ for coro in asyncio.as_completed(tasks):
+ result = await coro
+
+ if configs.show_all or result.is_visible(configs):
+ cat = result.category or cat_name
+ if cat not in printed_cats:
+ print(f"\n{Fore.MAGENTA}== {cat.upper()} SITES =={Style.RESET_ALL}")
+ printed_cats.add(cat)
+
+ result.show(configs)
+ results.append(result)
- for coro in asyncio.as_completed(tasks):
- result = await coro
-
- actual_cat = result.category or "Unknown"
- # Handle specific logic where skipping needs to happen early
- if configs.show_all or result.is_visible(configs):
- if printed_cats is not None and actual_cat not in printed_cats:
- print(f"\n{Fore.MAGENTA}== {actual_cat.upper()} SITES =={Style.RESET_ALL}")
- printed_cats.add(actual_cat)
-
- result.show(configs)
- results.append(result)
-
return results
@@ -152,6 +184,7 @@ def run_user_category(
username,
configs,
printed_cats=printed_cats,
+ cat_override=category_name,
)
)
@@ -202,17 +235,18 @@ async def _run_user_full_async(username: str, configs: ScanConfig) -> List[Resul
continue
if configs.show_all:
- print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")
- printed_cats.add(display_name)
+ if display_name not in printed_cats:
+ print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")
+ printed_cats.add(display_name)
for coro in asyncio.as_completed(tasks):
result = await coro
if configs.show_all or result.is_visible(configs):
- display_name = result.category or "Unknown"
- if display_name not in printed_cats:
- print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")
- printed_cats.add(display_name)
+ cat = result.category or display_name
+ if cat not in printed_cats:
+ print(f"\n{Fore.MAGENTA}== {cat.upper()} SITES =={Style.RESET_ALL}")
+ printed_cats.add(cat)
result.show(configs)
all_results.append(result)