🕵️♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.
Python cybersecuritycybersecurity-toolsemail-osintethical-hackingethical-hacking-toolsidentificationinfosecosintosint-emailosint-toolosint-toolsosint-usernamepentestingpythonredteam-toolsredteamingthreat-intelligence
fix(core): group multi-module scans by category and prevent header bleeding (#641)
Kaif687dff5c40f9
diff --git a/tests/test_email_orchestrator.py b/tests/test_email_orchestrator.pyindex b1d8db1..4f65b1e 100644--- a/tests/test_email_orchestrator.py+++ b/tests/test_email_orchestrator.py@@ -8,3 +8,60 @@ def test_set_concurrency():# restoreset_concurrency(original_max)+++def test_run_email_batch_multiple_categories_grouped(capsys, monkeypatch):+ import types+ from user_scanner.core import email_orchestrator+ from user_scanner.core.helpers import ScanConfig+ from user_scanner.core.result import Result++ mod1 = types.ModuleType("cat_a.site1")+ mod1.__file__ = "/path/to/email_scan/cat_a/site1.py"++ async def val1(e):+ return Result.taken(username=e)++ setattr(mod1, "validate_site1", val1)++ mod2 = types.ModuleType("cat_b.site2")+ mod2.__file__ = "/path/to/email_scan/cat_b/site2.py"++ async def val2(e):+ return Result.taken(username=e)++ setattr(mod2, "validate_site2", val2)++ mod3 = types.ModuleType("cat_a.site3")+ mod3.__file__ = "/path/to/email_scan/cat_a/site3.py"++ async def val3(e):+ return Result.taken(username=e)++ setattr(mod3, "validate_site3", val3)++ monkeypatch.setattr(email_orchestrator, "find_category", lambda m: m.__name__.split(".")[0].capitalize())+ monkeypatch.setattr(email_orchestrator, "get_site_name", lambda m: m.__name__.split(".")[-1].capitalize())++ results = email_orchestrator.run_email_module_batch([mod1, mod2, mod3], "[email protected]", ScanConfig(show_all=True))+ assert len(results) == 3++ out = capsys.readouterr().out+ assert "== CAT_A SITES ==" in out+ assert "== CAT_B SITES ==" in out++ cat_a_pos = out.find("== CAT_A SITES ==")+ cat_b_pos = out.find("== CAT_B SITES ==")+ assert cat_a_pos != -1 and cat_b_pos != -1+ assert cat_a_pos < cat_b_pos++ site1_pos = out.find("Site1")+ site2_pos = out.find("Site2")+ site3_pos = out.find("Site3")++ # Site1 and Site3 belong to Cat_a and must appear BEFORE Cat_b header+ assert cat_a_pos < site1_pos < cat_b_pos+ assert cat_a_pos < site3_pos < cat_b_pos+ # Site2 belongs to Cat_b and must appear AFTER Cat_b header+ assert site2_pos > cat_b_pos+diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.pyindex 588b0bc..3f8efec 100644--- a/tests/test_orchestrator.py+++ b/tests/test_orchestrator.py@@ -62,3 +62,43 @@ def test_set_concurrency():# restoreset_concurrency(original_max)+++def test_run_batch_multiple_categories_grouped(capsys, monkeypatch):+ mod1 = types.ModuleType("cat_a.site1")+ mod1.__file__ = "/path/to/user_scan/cat_a/site1.py"+ setattr(mod1, "validate_site1", lambda u: Result.taken(username=u))++ mod2 = types.ModuleType("cat_b.site2")+ mod2.__file__ = "/path/to/user_scan/cat_b/site2.py"+ setattr(mod2, "validate_site2", lambda u: Result.taken(username=u))++ mod3 = types.ModuleType("cat_a.site3")+ mod3.__file__ = "/path/to/user_scan/cat_a/site3.py"+ setattr(mod3, "validate_site3", lambda u: Result.taken(username=u))++ monkeypatch.setattr(orchestrator, "find_category", lambda m: m.__name__.split(".")[0].capitalize())+ monkeypatch.setattr(orchestrator, "get_site_name", lambda m: m.__name__.split(".")[-1].capitalize())++ results = orchestrator.run_user_module([mod1, mod2, mod3], "alice", ScanConfig(show_all=True))+ assert len(results) == 3++ out = capsys.readouterr().out+ assert "== CAT_A SITES ==" in out+ assert "== CAT_B SITES ==" in out++ cat_a_pos = out.find("== CAT_A SITES ==")+ cat_b_pos = out.find("== CAT_B SITES ==")+ assert cat_a_pos != -1 and cat_b_pos != -1+ assert cat_a_pos < cat_b_pos++ site1_pos = out.find("Site1")+ site2_pos = out.find("Site2")+ site3_pos = out.find("Site3")++ # Site1 and Site3 belong to Cat_a and must appear BEFORE Cat_b header+ assert cat_a_pos < site1_pos < cat_b_pos+ assert cat_a_pos < site3_pos < cat_b_pos+ # Site2 belongs to Cat_b and must appear AFTER Cat_b header+ assert site2_pos > cat_b_pos+diff --git a/user_scanner/core/email_orchestrator.py b/user_scanner/core/email_orchestrator.pyindex 6fa4403..9c16edd 100644--- a/user_scanner/core/email_orchestrator.py+++ b/user_scanner/core/email_orchestrator.py@@ -2,7 +2,7 @@ import asyncioimport httpxfrom pathlib import Pathfrom types import ModuleType-from typing import List, Optional, Set, Union, Callable+from typing import Callable, Dict, List, Optional, Set, Unionfrom colorama import Fore, Style@@ -65,6 +65,7 @@ async def _async_worker(sem: asyncio.Semaphore,configs: ScanConfig,printed_cats: Optional[Set] = None,+ cat_override: Optional[str] = None,on_start: Optional[Callable[[str], None]] = None,) -> Result:async with sem:@@ -72,7 +73,7 @@ async def _async_worker(if on_start:on_start(site_name)func = get_scan_func(module)- actual_cat = find_category(module) or "Email"+ actual_cat = cat_override or find_category(module) or "Email"params = {"site_name": site_name.capitalize(),@@ -109,13 +110,26 @@ async def _run_batch(email: str,configs: ScanConfig,printed_cats: Optional[Set] = None,+ cat_override: Optional[str] = None,+ sem: Optional[asyncio.Semaphore] = None,) -> List[Result]:if not modules:return []- sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)+ if sem is None:+ sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)++ if printed_cats is None:+ printed_cats = set()+results = []-++ category_map: Dict[str, List[ModuleType]] = {}+ for module in modules:+ cat = cat_override or find_category(module) or "Email"+ display_cat = cat.capitalize()+ category_map.setdefault(display_cat, []).append(module)+with Progress(SpinnerColumn(),TextColumn("[progress.description]{task.description}"),@@ -129,33 +143,49 @@ async def _run_batch(def on_start_cb(site: str):progress.update(task_id, description=f"[cyan]Scanning {email}... ({site})")- tasks = []- for module in modules:- t = asyncio.create_task(- _async_worker(- module,- email,- sem,- configs,- printed_cats=printed_cats,- on_start=on_start_cb,+ # 1. Pre-spawn all tasks for all categories (global concurrency)+ spawned_category_tasks = []+ for cat_name, cat_modules in category_map.items():+ tasks = []+ for module in cat_modules:+ t = asyncio.create_task(+ _async_worker(+ module,+ email,+ sem,+ configs,+ cat_override=cat_name,+ on_start=on_start_cb,+ ))- )- t.add_done_callback(lambda t: progress.advance(task_id))- tasks.append(t)-- for coro in asyncio.as_completed(tasks):- result = await coro- actual_cat = result.category or "Unknown"- if configs.show_all or result.is_visible(configs):- if printed_cats is not None and actual_cat not in printed_cats:+ t.add_done_callback(lambda t: progress.advance(task_id))+ tasks.append(t)+ spawned_category_tasks.append((cat_name, tasks))++ # 2. Await tasks category by category to stream grouped output+ for cat_name, tasks in spawned_category_tasks:+ if not tasks:+ continue++ if configs.show_all:+ if cat_name not in printed_cats:print(- f"\n{Fore.MAGENTA}== {actual_cat.upper()} SITES =={Style.RESET_ALL}"+ f"\n{Fore.MAGENTA}== {cat_name.upper()} SITES =={Style.RESET_ALL}")- printed_cats.add(actual_cat)+ printed_cats.add(cat_name)- result.show(configs)- results.append(result)+ for coro in asyncio.as_completed(tasks):+ result = await coro+ if configs.show_all or result.is_visible(configs):+ cat = result.category or cat_name+ if cat not in printed_cats:+ print(+ f"\n{Fore.MAGENTA}== {cat.upper()} SITES =={Style.RESET_ALL}"+ )+ printed_cats.add(cat)++ result.show(configs)+ results.append(result)return results@@ -195,6 +225,7 @@ async def _run_email_category_batch_async(email,configs,printed_cats=printed_cats,+ cat_override=cat_name,)def run_email_category_batch(@@ -246,7 +277,7 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[Remail,sem,configs,- printed_cats=printed_cats,+ cat_override=display_name,on_start=on_start_cb,))@@ -259,18 +290,19 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[Rcontinueif configs.show_all:- print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")- printed_cats.add(display_name)+ if display_name not in printed_cats:+ print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")+ printed_cats.add(display_name)for coro in asyncio.as_completed(tasks):result = await coroif configs.show_all or result.is_visible(configs):- display_name = result.category or "Unknown"- if display_name not in printed_cats:+ cat = result.category or display_name+ if cat not in printed_cats:print(- f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}"+ f"\n{Fore.MAGENTA}== {cat.upper()} SITES =={Style.RESET_ALL}")- printed_cats.add(display_name)+ printed_cats.add(cat)result.show(configs)all_results.append(result)diff --git a/user_scanner/core/orchestrator.py b/user_scanner/core/orchestrator.pyindex 6d511a7..c481b80 100644--- a/user_scanner/core/orchestrator.py+++ b/user_scanner/core/orchestrator.py@@ -84,13 +84,25 @@ async def _run_batch(configs: ScanConfig,printed_cats: Optional[Set] = None,cat_override: Optional[str] = None,- sem: Optional[asyncio.Semaphore] = None+ sem: Optional[asyncio.Semaphore] = None,) -> List[Result]:+ if not modules:+ return []+if sem is None:sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)-++ if printed_cats is None:+ printed_cats = set()+results = []-++ category_map: Dict[str, List[ModuleType]] = {}+ for module in modules:+ cat = cat_override or find_category(module) or "Unknown"+ display_cat = cat.capitalize()+ category_map.setdefault(display_cat, []).append(module)+with Progress(SpinnerColumn(),TextColumn("[progress.description]{task.description}"),@@ -104,27 +116,47 @@ async def _run_batch(def on_start_cb(site: str):progress.update(task_id, description=f"[cyan]Scanning {username}... ({site})")- tasks = []- for module in modules:- t = asyncio.create_task(- _async_worker(module, username, sem, configs, cat_override=cat_override, on_start=on_start_cb)- )- t.add_done_callback(lambda t: progress.advance(task_id))- tasks.append(t)+ # 1. Pre-spawn all tasks for all categories (global concurrency)+ spawned_category_tasks = []+ for cat_name, cat_modules in category_map.items():+ tasks = []+ for module in cat_modules:+ t = asyncio.create_task(+ _async_worker(+ module,+ username,+ sem,+ configs,+ cat_override=cat_name,+ on_start=on_start_cb,+ )+ )+ t.add_done_callback(lambda t: progress.advance(task_id))+ tasks.append(t)+ spawned_category_tasks.append((cat_name, tasks))++ # 2. Await tasks category by category to stream grouped output+ for cat_name, tasks in spawned_category_tasks:+ if not tasks:+ continue++ if configs.show_all:+ if cat_name not in printed_cats:+ print(f"\n{Fore.MAGENTA}== {cat_name.upper()} SITES =={Style.RESET_ALL}")+ printed_cats.add(cat_name)++ for coro in asyncio.as_completed(tasks):+ result = await coro++ if configs.show_all or result.is_visible(configs):+ cat = result.category or cat_name+ if cat not in printed_cats:+ print(f"\n{Fore.MAGENTA}== {cat.upper()} SITES =={Style.RESET_ALL}")+ printed_cats.add(cat)++ result.show(configs)+ results.append(result)- for coro in asyncio.as_completed(tasks):- result = await coro-- actual_cat = result.category or "Unknown"- # Handle specific logic where skipping needs to happen early- if configs.show_all or result.is_visible(configs):- if printed_cats is not None and actual_cat not in printed_cats:- print(f"\n{Fore.MAGENTA}== {actual_cat.upper()} SITES =={Style.RESET_ALL}")- printed_cats.add(actual_cat)-- result.show(configs)- results.append(result)-return results@@ -152,6 +184,7 @@ def run_user_category(username,configs,printed_cats=printed_cats,+ cat_override=category_name,))@@ -202,17 +235,18 @@ async def _run_user_full_async(username: str, configs: ScanConfig) -> List[Resulcontinueif configs.show_all:- print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")- printed_cats.add(display_name)+ if display_name not in printed_cats:+ print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")+ printed_cats.add(display_name)for coro in asyncio.as_completed(tasks):result = await coroif configs.show_all or result.is_visible(configs):- display_name = result.category or "Unknown"- if display_name not in printed_cats:- print(f"\n{Fore.MAGENTA}== {display_name.upper()} SITES =={Style.RESET_ALL}")- printed_cats.add(display_name)+ cat = result.category or display_name+ if cat not in printed_cats:+ print(f"\n{Fore.MAGENTA}== {cat.upper()} SITES =={Style.RESET_ALL}")+ printed_cats.add(cat)result.show(configs)all_results.append(result)