π΅οΈββοΈ (2-in-1) Email & Username OSINT suite featuring native MCP support for deep data extraction just from a single Email/Username. Analyzes 2720+ actively maintained scan vectors (210+ email / 2510+ username) for security research, investigations, and digital footprinting.
Python cybersecuritycybersecurity-toolsemail-osintethical-hackingethical-hacking-toolsidentificationinfosecosintosint-emailosint-toolosint-toolsosint-usernamepentestingpythonredteam-toolsredteamingthreat-intelligence
fix(proxy): check proxies over HTTPS and call them reachable, not working (#729) (#745)
jkfdced6742435
diff --git a/README.md b/README.mdindex 12b8115..5633e51 100644--- a/README.md+++ b/README.md@@ -86,7 +86,7 @@ With **2720+ total scan vectors**βincluding **210+ email-integrated sites** an- β‘ **High-Throughput Parallel Engine:** Powered by `httpx` and `curl_cffi` for maximum concurrency with automated TLS fingerprint impersonation.- π **Permutation & Alias Generator:** Wildcard-based username variation generation to catch typosquatting or alternative aliases.- π **Multi-Format Reports:** Automated exports to **PDF** (with profile photos), **JSON**, and **CSV** for pipeline integration.-- π **Advanced Proxy Pivoting:** Built-in proxy rotation with protocol auto-detection (`http`, `socks5`) and pre-scan health validation (`--validate-proxies`).+- π **Advanced Proxy Pivoting:** Built-in proxy rotation with protocol auto-detection (`http`, `socks5`) and pre-scan reachability check (`--validate-proxies`).- π¨ **Responsive Terminal UI:** Dynamic progress tracking, self-adaptive category grids (`-lu`/`-le`), and clear status reporting.---@@ -202,7 +202,7 @@ user-scanner -u johndoe -f json -o results.json# Verbose URL reporting and show all results (including not found)user-scanner -u johndoe -v --all-# Rotate proxies with pre-scan validation check+# Rotate proxies with a pre-scan reachability checkuser-scanner -u johndoe -P proxies.txt --validate-proxies```diff --git a/docs/FLAGS.md b/docs/FLAGS.mdindex f7dd14c..31cc268 100644--- a/docs/FLAGS.md+++ b/docs/FLAGS.md@@ -24,7 +24,7 @@| `-m, --module MODULE` | Scan a specific module (comma-separated for multiple); also narrows `--cross-scan` || `-p, --permute PERMUTE` | Generate username permutations using a pattern/suffix || `-P, --proxy-file FILE` | Use proxies from file (one per line). Tip: Combine with `--no-pdf-media` to save proxy bandwidth |-| `--validate-proxies` | Validate proxies before scanning (tests against google.com) |+| `--validate-proxies` | Check that proxies are reachable before scanning (HTTPS request to gstatic.com/generate_204; not a trust check) || `-s, --stop STOP` | Limit the number of permutations generated (at least 1) || `-d, --delay DELAY` | Delay (in seconds) between requests, 0 to 3600 || `-t, --timeout TIMEOUT` | Override default request timeout in seconds, above 0 and at most 3600 |diff --git a/tests/test_helpers.py b/tests/test_helpers.pyindex 777640a..dc0d615 100644--- a/tests/test_helpers.py+++ b/tests/test_helpers.py@@ -339,6 +339,42 @@ def test_validate_proxy_single_worker(mock_client):assert result == proxies+def test_validate_proxy_rejects_proxy_that_cannot_tunnel_tls():+ # Answers every plaintext request with a forged 204 but refuses CONNECT,+ # so it cannot carry an HTTPS request end to end.+ import socket+ import threading++ server = socket.socket()+ server.bind(("127.0.0.1", 0))+ server.listen(8)+ port = server.getsockname()[1]+ requests: list[bytes] = []++ def serve():+ while True:+ try:+ conn, _ = server.accept()+ except OSError:+ return+ with conn:+ data = conn.recv(4096)+ requests.append(data)+ if data.startswith(b"CONNECT"):+ conn.sendall(b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\n\r\n")+ else:+ conn.sendall(b"HTTP/1.1 204 No Content\r\n\r\n")++ threading.Thread(target=serve, daemon=True).start()+ try:+ result = helpers.validate_proxies([f"http://127.0.0.1:{port}"], timeout=2)+ finally:+ server.close()++ assert result == []+ assert requests and all(r.startswith(b"CONNECT") for r in requests)++def test_proxy_manager_add_protocol(tmp_path):proxy_file = tmp_path / "test_proxies.txt"proxy_file.write_text("""proxy1.example.com:8080diff --git a/user_scanner/__main__.py b/user_scanner/__main__.pyindex 6104436..f4e12a1 100644--- a/user_scanner/__main__.py+++ b/user_scanner/__main__.py@@ -196,7 +196,7 @@ def main():parser.add_argument("--validate-proxies",action="store_true",- help="Validate proxies before scanning (tests against gstatic.com/generate_204)",+ help="Check that proxies are reachable before scanning (HTTPS request to gstatic.com/generate_204; not a trust check)",)parser.add_argument(@@ -423,16 +423,16 @@ def main():working_proxies = validate_proxies(all_proxies)if not working_proxies:- print(f"{R}[β] No working proxies found{X}")+ print(f"{R}[β] No reachable proxies found{X}")sys.exit(1)print(- f"{G}[+] Found {len(working_proxies)} working proxies out of {len(all_proxies)}{X}"+ f"{G}[+] Found {len(working_proxies)} reachable proxies out of {len(all_proxies)}{X}")set_proxy_manager(proxies=working_proxies)proxy_count = get_proxy_count()- print(f"{G}[+] Using {proxy_count} validated proxies{X}")+ print(f"{G}[+] Using {proxy_count} reachable proxies{X}")else:set_proxy_manager(args.proxy_file)proxy_count = get_proxy_count()diff --git a/user_scanner/core/helpers.py b/user_scanner/core/helpers.pyindex 39de27e..1b976e1 100644--- a/user_scanner/core/helpers.py+++ b/user_scanner/core/helpers.py@@ -577,7 +577,9 @@ def find_category(module: ModuleType) -> str | None:async def _validate_proxy_async(proxy: str, timeout: int) -> Optional[str]:try:async with httpx.AsyncClient(proxy=proxy, timeout=timeout) as client:- response = await client.get("http://gstatic.com/generate_204")+ # HTTPS with certificate verification (httpx default), so the+ # proxy has to tunnel TLS end to end instead of forging a reply.+ response = await client.get("https://www.gstatic.com/generate_204")if response.status_code in (200, 204):return proxyexcept Exception:@@ -601,7 +603,11 @@ async def _validate_proxies_batch(proxy_list: List[str], timeout: int, max_workereturn working_proxiesdef validate_proxies(proxy_list: List[str], timeout: int = 5, max_workers: int = 50) -> List[str]:- """Validate proxies by testing them against gstatic.com/generate_204. Returns list of working proxies."""+ """Return the proxies that can reach https://www.gstatic.com/generate_204.++ This only checks that a proxy is reachable and tunnels TLS. It says+ nothing about whether the proxy is trustworthy.+ """return asyncio.run(_validate_proxies_batch(proxy_list, timeout, max_workers))