🕵️♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.
Python cybersecuritycybersecurity-toolsemail-osintethical-hackingethical-hacking-toolsidentificationinfosecosintosint-emailosint-toolosint-toolsosint-usernamepentestingpythonredteam-toolsredteamingthreat-intelligence
fix: resolve core engine concurrency hangs by introducing per-key session locks, strict execution timeouts, and dynamic thread pool scaling
Kaifed321aac634d
diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.pyindex d958876..588b0bc 100644--- a/tests/test_orchestrator.py+++ b/tests/test_orchestrator.py@@ -58,7 +58,7 @@ def test_set_concurrency():original_max = orchestrator.MAX_CONCURRENT_REQUESTSset_concurrency(10)assert orchestrator.MAX_CONCURRENT_REQUESTS == 10- assert orchestrator._shared_executor._max_workers == 10+ assert orchestrator._shared_executor._max_workers == max(10 * 2, 250)# restoreset_concurrency(original_max)diff --git a/user_scanner/core/email_orchestrator.py b/user_scanner/core/email_orchestrator.pyindex b714d89..d45bcd9 100644--- a/user_scanner/core/email_orchestrator.py+++ b/user_scanner/core/email_orchestrator.py@@ -88,10 +88,13 @@ async def _async_worker(try:import inspect+ module_timeout = (get_global_timeout() or 15.0) + 10.0if inspect.iscoroutinefunction(func):- result = await func(email)+ result = await asyncio.wait_for(func(email), timeout=module_timeout)else:- result = await asyncio.to_thread(func, email)+ result = await asyncio.wait_for(asyncio.to_thread(func, email), timeout=module_timeout)+ except asyncio.TimeoutError:+ result = Result.error(f"Module execution timed out after {module_timeout}s")except Exception as e:result = Result.error(e)@@ -108,12 +111,14 @@ async def _run_batch(tasks = []for module in modules:tasks.append(- _async_worker(- module,- email,- sem,- configs,- printed_cats=printed_cats,+ asyncio.create_task(+ _async_worker(+ module,+ email,+ sem,+ configs,+ printed_cats=printed_cats,+ )))@@ -132,6 +137,9 @@ async def _run_batch() as progress:task_id = progress.add_task(f"[cyan]Scanning {email}...", total=len(tasks))+ for task in tasks:+ task.add_done_callback(lambda t: progress.advance(task_id))+for coro in asyncio.as_completed(tasks):result = await coroactual_cat = result.category or "Unknown"@@ -144,7 +152,6 @@ async def _run_batch(result.show(configs)results.append(result)- progress.advance(task_id)return results@@ -152,6 +159,9 @@ async def _run_batch(async def _run_email_module_batch_async(module: Union[ModuleType, List[ModuleType]], email: str, configs: ScanConfig) -> List[Result]:+ loop = asyncio.get_running_loop()+ import concurrent.futures+ loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=250))modules = [module] if isinstance(module, ModuleType) else list(module)return await _run_batch(modules, email, configs)@@ -165,6 +175,9 @@ def run_email_module_batch(async def _run_email_category_batch_async(category_path: Path, email: str, configs: ScanConfig) -> List[Result]:+ loop = asyncio.get_running_loop()+ import concurrent.futures+ loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=250))cat_name = category_path.stem.capitalize()modules = load_modules(category_path)printed_cats = set()@@ -187,6 +200,9 @@ def run_email_category_batch(async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[Result]:+ loop = asyncio.get_running_loop()+ import concurrent.futures+ loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=250))categories = load_categories(True, configs.no_nsfw)all_results = []printed_cats: Set[str] = set()@@ -194,20 +210,23 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[R# 1. Pre-spawn all tasks for all categories (global concurrency)category_tasks = []total_tasks = 0+ sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)+for cat_name, cat_path in categories.items():display_name = cat_name.capitalize()modules = load_modules(cat_path)- sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)tasks = []for module in modules:tasks.append(- _async_worker(- module,- email,- sem,- configs,- printed_cats=printed_cats,+ asyncio.create_task(+ _async_worker(+ module,+ email,+ sem,+ configs,+ printed_cats=printed_cats,+ )))category_tasks.append((display_name, tasks))@@ -224,6 +243,10 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[R) as progress:task_id = progress.add_task(f"[cyan]Scanning {email}...", total=total_tasks)+ for _, tasks in category_tasks:+ for task in tasks:+ task.add_done_callback(lambda t: progress.advance(task_id))+for display_name, tasks in category_tasks:if not tasks:continue@@ -244,7 +267,6 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[Rresult.show(configs)all_results.append(result)- progress.advance(task_id)return all_resultsdiff --git a/user_scanner/core/impersonate.py b/user_scanner/core/impersonate.pyindex aaa964e..a600afa 100644--- a/user_scanner/core/impersonate.py+++ b/user_scanner/core/impersonate.py@@ -11,6 +11,7 @@ DEFAULT_IMPERSONATE = "chrome"DEFAULT_TIMEOUT = 15.0_sessions: dict[tuple, cffi.Session] = {}+_key_locks: dict[tuple, threading.Lock] = {}_warmed: set[tuple] = set()_lock = threading.Lock()@@ -93,12 +94,20 @@ def _get_warm_session(proxies={"http": proxy, "https": proxy} if proxy else None,)_sessions[key] = session-- if warmup_url and key not in _warmed:- # A blocked (403) warm-up still returns normally and sets the cookie;- # only a network error leaves the session unwarmed for a later retry.- session.get(warmup_url, timeout=_timeout())- _warmed.add(key)+ _key_locks[key] = threading.Lock()++ key_lock = _key_locks.get(key)+ if key_lock is None:+ key_lock = threading.Lock()+ _key_locks[key] = key_lock++ if warmup_url and key not in _warmed:+ with key_lock:+ if key not in _warmed:+ # A blocked (403) warm-up still returns normally and sets the cookie;+ # only a network error leaves the session unwarmed for a later retry.+ session.get(warmup_url, timeout=_timeout())+ _warmed.add(key)return sessiondiff --git a/user_scanner/core/orchestrator.py b/user_scanner/core/orchestrator.pyindex 3c2a74c..1cd1a6b 100644--- a/user_scanner/core/orchestrator.py+++ b/user_scanner/core/orchestrator.py@@ -25,12 +25,12 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, MofNCoMAX_CONCURRENT_REQUESTS = 60-_shared_executor = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS)+_shared_executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(MAX_CONCURRENT_REQUESTS * 2, 250))def set_concurrency(val: int):global MAX_CONCURRENT_REQUESTS, _shared_executorMAX_CONCURRENT_REQUESTS = val- _shared_executor = concurrent.futures.ThreadPoolExecutor(max_workers=val)+ _shared_executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(val * 2, 250))async def _async_worker(module: ModuleType,@@ -58,11 +58,17 @@ async def _async_worker(return Result.skipped().update(**params)try:+ module_timeout = (get_global_timeout() or 15.0) + 10.0if inspect.iscoroutinefunction(func):- result = await func(username)+ result = await asyncio.wait_for(func(username), timeout=module_timeout)else:loop = asyncio.get_running_loop()- result = await loop.run_in_executor(_shared_executor, func, username)+ result = await asyncio.wait_for(+ loop.run_in_executor(_shared_executor, func, username),+ timeout=module_timeout+ )+ except asyncio.TimeoutError:+ result = Result.error(f"Module execution timed out after {module_timeout}s")except Exception as e:result = Result.error(e)@@ -100,6 +106,9 @@ async def _run_batch() as progress:task_id = progress.add_task(f"[cyan]Scanning {username}...", total=len(tasks))+ for task in tasks:+ task.add_done_callback(lambda t: progress.advance(task_id))+for coro in asyncio.as_completed(tasks):result = await coro@@ -112,7 +121,6 @@ async def _run_batch(result.show(configs)results.append(result)- progress.advance(task_id)return results@@ -179,6 +187,10 @@ async def _run_user_full_async(username: str, configs: ScanConfig) -> List[Resul) as progress:task_id = progress.add_task(f"[cyan]Scanning {username}...", total=total_tasks)+ for _, tasks in category_tasks:+ for task in tasks:+ task.add_done_callback(lambda t: progress.advance(task_id))+for display_name, tasks in category_tasks:if not tasks:continue@@ -198,7 +210,6 @@ async def _run_user_full_async(username: str, configs: ScanConfig) -> List[Resulresult.show(configs)all_results.append(result)- progress.advance(task_id)return all_results