Skip to content

🕵️‍♂️ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.

fix: resolve core engine concurrency hangs by introducing per-key session locks, strict execution timeouts, and dynamic thread pool scaling

Kaifed321aac634d
diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py
index d958876..588b0bc 100644
--- a/tests/test_orchestrator.py
+++ b/tests/test_orchestrator.py
@@ -58,7 +58,7 @@ def test_set_concurrency():
original_max = orchestrator.MAX_CONCURRENT_REQUESTS
set_concurrency(10)
assert orchestrator.MAX_CONCURRENT_REQUESTS == 10
- assert orchestrator._shared_executor._max_workers == 10
+ assert orchestrator._shared_executor._max_workers == max(10 * 2, 250)
# restore
set_concurrency(original_max)
diff --git a/user_scanner/core/email_orchestrator.py b/user_scanner/core/email_orchestrator.py
index b714d89..d45bcd9 100644
--- a/user_scanner/core/email_orchestrator.py
+++ b/user_scanner/core/email_orchestrator.py
@@ -88,10 +88,13 @@ async def _async_worker(
try:
import inspect
+ module_timeout = (get_global_timeout() or 15.0) + 10.0
if inspect.iscoroutinefunction(func):
- result = await func(email)
+ result = await asyncio.wait_for(func(email), timeout=module_timeout)
else:
- result = await asyncio.to_thread(func, email)
+ result = await asyncio.wait_for(asyncio.to_thread(func, email), timeout=module_timeout)
+ except asyncio.TimeoutError:
+ result = Result.error(f"Module execution timed out after {module_timeout}s")
except Exception as e:
result = Result.error(e)
@@ -108,12 +111,14 @@ async def _run_batch(
tasks = []
for module in modules:
tasks.append(
- _async_worker(
- module,
- email,
- sem,
- configs,
- printed_cats=printed_cats,
+ asyncio.create_task(
+ _async_worker(
+ module,
+ email,
+ sem,
+ configs,
+ printed_cats=printed_cats,
+ )
)
)
@@ -132,6 +137,9 @@ async def _run_batch(
) as progress:
task_id = progress.add_task(f"[cyan]Scanning {email}...", total=len(tasks))
+ for task in tasks:
+ task.add_done_callback(lambda t: progress.advance(task_id))
+
for coro in asyncio.as_completed(tasks):
result = await coro
actual_cat = result.category or "Unknown"
@@ -144,7 +152,6 @@ async def _run_batch(
result.show(configs)
results.append(result)
- progress.advance(task_id)
return results
@@ -152,6 +159,9 @@ async def _run_batch(
async def _run_email_module_batch_async(
module: Union[ModuleType, List[ModuleType]], email: str, configs: ScanConfig
) -> List[Result]:
+ loop = asyncio.get_running_loop()
+ import concurrent.futures
+ loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=250))
modules = [module] if isinstance(module, ModuleType) else list(module)
return await _run_batch(modules, email, configs)
@@ -165,6 +175,9 @@ def run_email_module_batch(
async def _run_email_category_batch_async(
category_path: Path, email: str, configs: ScanConfig
) -> List[Result]:
+ loop = asyncio.get_running_loop()
+ import concurrent.futures
+ loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=250))
cat_name = category_path.stem.capitalize()
modules = load_modules(category_path)
printed_cats = set()
@@ -187,6 +200,9 @@ def run_email_category_batch(
async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[Result]:
+ loop = asyncio.get_running_loop()
+ import concurrent.futures
+ loop.set_default_executor(concurrent.futures.ThreadPoolExecutor(max_workers=250))
categories = load_categories(True, configs.no_nsfw)
all_results = []
printed_cats: Set[str] = set()
@@ -194,20 +210,23 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[R
# 1. Pre-spawn all tasks for all categories (global concurrency)
category_tasks = []
total_tasks = 0
+ sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+
for cat_name, cat_path in categories.items():
display_name = cat_name.capitalize()
modules = load_modules(cat_path)
- sem = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
tasks = []
for module in modules:
tasks.append(
- _async_worker(
- module,
- email,
- sem,
- configs,
- printed_cats=printed_cats,
+ asyncio.create_task(
+ _async_worker(
+ module,
+ email,
+ sem,
+ configs,
+ printed_cats=printed_cats,
+ )
)
)
category_tasks.append((display_name, tasks))
@@ -224,6 +243,10 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[R
) as progress:
task_id = progress.add_task(f"[cyan]Scanning {email}...", total=total_tasks)
+ for _, tasks in category_tasks:
+ for task in tasks:
+ task.add_done_callback(lambda t: progress.advance(task_id))
+
for display_name, tasks in category_tasks:
if not tasks:
continue
@@ -244,7 +267,6 @@ async def _run_email_full_batch_async(email: str, configs: ScanConfig) -> List[R
result.show(configs)
all_results.append(result)
- progress.advance(task_id)
return all_results
diff --git a/user_scanner/core/impersonate.py b/user_scanner/core/impersonate.py
index aaa964e..a600afa 100644
--- a/user_scanner/core/impersonate.py
+++ b/user_scanner/core/impersonate.py
@@ -11,6 +11,7 @@ DEFAULT_IMPERSONATE = "chrome"
DEFAULT_TIMEOUT = 15.0
_sessions: dict[tuple, cffi.Session] = {}
+_key_locks: dict[tuple, threading.Lock] = {}
_warmed: set[tuple] = set()
_lock = threading.Lock()
@@ -93,12 +94,20 @@ def _get_warm_session(
proxies={"http": proxy, "https": proxy} if proxy else None,
)
_sessions[key] = session
-
- if warmup_url and key not in _warmed:
- # A blocked (403) warm-up still returns normally and sets the cookie;
- # only a network error leaves the session unwarmed for a later retry.
- session.get(warmup_url, timeout=_timeout())
- _warmed.add(key)
+ _key_locks[key] = threading.Lock()
+
+ key_lock = _key_locks.get(key)
+ if key_lock is None:
+ key_lock = threading.Lock()
+ _key_locks[key] = key_lock
+
+ if warmup_url and key not in _warmed:
+ with key_lock:
+ if key not in _warmed:
+ # A blocked (403) warm-up still returns normally and sets the cookie;
+ # only a network error leaves the session unwarmed for a later retry.
+ session.get(warmup_url, timeout=_timeout())
+ _warmed.add(key)
return session
diff --git a/user_scanner/core/orchestrator.py b/user_scanner/core/orchestrator.py
index 3c2a74c..1cd1a6b 100644
--- a/user_scanner/core/orchestrator.py
+++ b/user_scanner/core/orchestrator.py
@@ -25,12 +25,12 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, MofNCo
MAX_CONCURRENT_REQUESTS = 60
-_shared_executor = concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS)
+_shared_executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(MAX_CONCURRENT_REQUESTS * 2, 250))
def set_concurrency(val: int):
global MAX_CONCURRENT_REQUESTS, _shared_executor
MAX_CONCURRENT_REQUESTS = val
- _shared_executor = concurrent.futures.ThreadPoolExecutor(max_workers=val)
+ _shared_executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(val * 2, 250))
async def _async_worker(
module: ModuleType,
@@ -58,11 +58,17 @@ async def _async_worker(
return Result.skipped().update(**params)
try:
+ module_timeout = (get_global_timeout() or 15.0) + 10.0
if inspect.iscoroutinefunction(func):
- result = await func(username)
+ result = await asyncio.wait_for(func(username), timeout=module_timeout)
else:
loop = asyncio.get_running_loop()
- result = await loop.run_in_executor(_shared_executor, func, username)
+ result = await asyncio.wait_for(
+ loop.run_in_executor(_shared_executor, func, username),
+ timeout=module_timeout
+ )
+ except asyncio.TimeoutError:
+ result = Result.error(f"Module execution timed out after {module_timeout}s")
except Exception as e:
result = Result.error(e)
@@ -100,6 +106,9 @@ async def _run_batch(
) as progress:
task_id = progress.add_task(f"[cyan]Scanning {username}...", total=len(tasks))
+ for task in tasks:
+ task.add_done_callback(lambda t: progress.advance(task_id))
+
for coro in asyncio.as_completed(tasks):
result = await coro
@@ -112,7 +121,6 @@ async def _run_batch(
result.show(configs)
results.append(result)
- progress.advance(task_id)
return results
@@ -179,6 +187,10 @@ async def _run_user_full_async(username: str, configs: ScanConfig) -> List[Resul
) as progress:
task_id = progress.add_task(f"[cyan]Scanning {username}...", total=total_tasks)
+ for _, tasks in category_tasks:
+ for task in tasks:
+ task.add_done_callback(lambda t: progress.advance(task_id))
+
for display_name, tasks in category_tasks:
if not tasks:
continue
@@ -198,7 +210,6 @@ async def _run_user_full_async(username: str, configs: ScanConfig) -> List[Resul
result.show(configs)
all_results.append(result)
- progress.advance(task_id)
return all_results