Self-hosted OSINT exposure checker — is your email, username, phone or password leaked? Keyless sources: breaches, infostealer logs, COMB credentials, HIBP k-anonymity, 200-platform username grid. Docker one-command install, no API keys, no AI, Apache-2.0.
PHP breach-checkcredential-exposuredata-breachdigital-footprintemail-osintfree-osinthaveibeenpwnedip-reputationleaked-credentialsleaked-dataosintosint-apiosint-toolsphone-lookupprivacy-toolspublic-datapwned-passwordsself-hostedusername-checkusername-osint
sync(ec): crt.sh: Cert Spotter fallback; mailcheck: api.mailcheck.ai host + 24h cache + stale grace (P4-S3 fixes)
Rana Khan61f286e06625
diff --git a/app/Modules/Osint/Sources/CrtShSource.php b/app/Modules/Osint/Sources/CrtShSource.phpindex 31e8dd0..fd32a9b 100644--- a/app/Modules/Osint/Sources/CrtShSource.php+++ b/app/Modules/Osint/Sources/CrtShSource.php@@ -7,14 +7,28 @@ namespace App\Modules\Osint\Sources;use Illuminate\Http\Client\Response;/**- * crt.sh certificate-transparency subdomain discovery (keyless).- * GET https://crt.sh/?q=%.{domain}&output=json&limit=200 — every subdomain- * that ever obtained a TLS certificate. Counts + names of the first 40.- * crt.sh can be slow; 8s timeout, graceful unavailable on failure.+ * Certificate-transparency subdomain discovery (keyless), P4-S3 hardened.+ *+ * crt.sh (`GET https://crt.sh/?q=%.{domain}&output=json&limit=200`) is the+ * classic keyless CT source but is frequently slow (>9s pool budget) or+ * returns 502 under load — 1-in-4 domain checks degraded to "unavailable"+ * and the admin health panel showed the tool rose.+ *+ * P4-S3 (2026-10-27) dual-endpoint strategy: BOTH requests are fired in the+ * same parallel Http::pool (zero added latency) and parse() uses whichever+ * answered:+ * crtsh.main — crt.sh JSON API (primary, unlimited but slow/flaky)+ * crtsh.alt — Cert Spotter public issuances API (fast 200ms, keyless,+ * 100 req/hour unauthenticated — fine at our volume, and the+ * 30-min per-identifier cache absorbs repeats)+ *+ * Output shape is unchanged (subdomains + subdomain_count), so the exposure+ * check UI and Trace reports keep working.*/class CrtShSource implements OsintSource{- private const URL = 'https://crt.sh/?q=';+ private const CRTSH_URL = 'https://crt.sh/?q=';+ private const CERTSPOTTER_URL = 'https://api.certspotter.com/v1/issuances';public function key(): string{@@ -38,42 +52,125 @@ class CrtShSource implements OsintSourcepublic function requests(string $type, string $identifier): array{+ $domain = strtolower($identifier);+return [['name' => $this->key() . '.main','method' => 'GET',- 'url' => self::URL . urlencode('%.' . strtolower($identifier)) . '&output=json&limit=200',- 'headers' => ['Accept' => 'application/json'],+ 'url' => self::CRTSH_URL . urlencode('%.' . $domain) . '&output=json&limit=200',+ 'headers' => [+ 'Accept' => 'application/json',+ // crt.sh intermittently 502s generic bot UAs; identify honestly.+ 'User-Agent' => '2mny-trace/1.0 (+https://2mny.com)',+ ],+ ],+ [+ 'name' => $this->key() . '.alt',+ 'method' => 'GET',+ 'url' => self::CERTSPOTTER_URL . '?domain=' . urlencode($domain) . '&include_subdomains=true&expand=dns_names',+ 'headers' => [+ 'Accept' => 'application/json',+ 'User-Agent' => '2mny-trace/1.0 (+https://2mny.com)',+ ],],];}public function parse(string $type, string $identifier, array $responses): array{- $resp = $responses[$this->key() . '.main'] ?? null;- if (!$resp instanceof Response || !$resp->ok()) {- return ['status' => 'unavailable', 'summary' => 'Certificate Transparency lookup timed out (crt.sh is slow at times).', 'data' => []];+ $identifier = strtolower($identifier);++ // ---- Primary: crt.sh JSON -------------------------------------------+ $main = $responses[$this->key() . '.main'] ?? null;+ if ($main instanceof Response && $main->ok()) {+ $j = $main->json();+ if (is_array($j) && $j !== []) {+ $names = $this->filterNames($this->namesFromCrtSh($j), $identifier);+ if ($names !== []) {+ return $this->result($names, 'crt.sh');+ }+ // Empty-but-valid crt.sh response: a legitimately unlisted+ // domain — still check the alternate before calling it clean.+ $alt = $this->parseAlt($responses, $identifier);+ if ($alt !== null) {+ return $alt;+ }++ return [+ 'status' => 'clean',+ 'summary' => 'No certificates found in Certificate Transparency logs for this domain.',+ 'severity' => 'info',+ 'data' => ['subdomains' => [], 'subdomain_count' => 0],+ ];+ }}- $j = $resp->json();- if (!is_array($j)) {- return ['status' => 'unavailable', 'summary' => 'Certificate Transparency returned an unexpected response.', 'data' => []];+ // ---- Fallback: Cert Spotter issuances -------------------------------+ $alt = $this->parseAlt($responses, $identifier);+ if ($alt !== null) {+ return $alt;}+ return ['status' => 'unavailable', 'summary' => 'Certificate Transparency lookup timed out (crt.sh is slow at times).', 'data' => []];+ }++ /**+ * @param array<int, mixed> $rows+ * @return array<int, string>+ */+ private function namesFromCrtSh(array $rows): array+ {$names = [];- foreach ($j as $row) {+ foreach ($rows as $row) {+ if (!is_array($row)) {+ continue;+ }$name = strtolower((string) ($row['name_value'] ?? ''));foreach (explode("\n", $name) as $n) {$n = trim($n, " *.\"\t");- if ($n !== '' && str_contains($n, $identifier)) {+ if ($n !== '') {+ $names[$n] = true;+ }+ }+ }++ return array_keys($names);+ }++ /**+ * Parse the Cert Spotter alternate response; null when it is unusable+ * (missing, failed, rate-limited or unparseable) so the caller can fall+ * through to honest-unavailable.+ */+ private function parseAlt(array $responses, string $identifier): ?array+ {+ $alt = $responses[$this->key() . '.alt'] ?? null;+ if (!$alt instanceof Response || !$alt->ok()) {+ return null;+ }++ $j = $alt->json();+ if (!is_array($j)) {+ return null;+ }++ $names = [];+ foreach ($j as $row) {+ if (!is_array($row) || !is_array($row['dns_names'] ?? null)) {+ continue;+ }+ foreach ($row['dns_names'] as $n) {+ $n = strtolower(trim((string) $n, " *.\"\t"));+ if ($n !== '') {$names[$n] = true;}}}- $names = array_keys($names);- sort($names);+ $names = $this->filterNames(array_keys($names), $identifier);if ($names === []) {+ // Valid response, no in-domain names: honest clean.return ['status' => 'clean','summary' => 'No certificates found in Certificate Transparency logs for this domain.',@@ -82,6 +179,35 @@ class CrtShSource implements OsintSource];}+ return $this->result($names, 'Cert Spotter CT mirror');+ }++ /**+ * Keep only names that belong to the queried domain (both endpoints can+ * return sibling noise), deduplicated + sorted.+ *+ * @param array<int, string> $names+ * @return array<int, string>+ */+ private function filterNames(array $names, string $identifier): array+ {+ $out = [];+ foreach ($names as $n) {+ if ($n === $identifier || str_ends_with($n, '.' . $identifier)) {+ $out[$n] = true;+ }+ }+ $out = array_keys($out);+ sort($out);++ return $out;+ }++ /**+ * @param array<int, string> $names+ */+ private function result(array $names, string $via): array+ {return ['status' => 'hit','summary' => count($names) . ' hostname(s) found in Certificate Transparency logs — this is the domain\'s public TLS footprint.',@@ -89,6 +215,7 @@ class CrtShSource implements OsintSource'data' => ['subdomain_count' => count($names),'subdomains' => array_slice($names, 0, 40),+ 'source' => $via,],];}diff --git a/app/Modules/Osint/Sources/MailcheckSource.php b/app/Modules/Osint/Sources/MailcheckSource.phpindex b2279fb..4f61a21 100644--- a/app/Modules/Osint/Sources/MailcheckSource.php+++ b/app/Modules/Osint/Sources/MailcheckSource.php@@ -5,19 +5,45 @@ declare(strict_types=1);namespace App\Modules\Osint\Sources;use Illuminate\Http\Client\Response;+use Illuminate\Support\Facades\Cache;/*** mailcheck.ai - keyless email-domain intelligence.- * GET https://mailcheck.ai/domain/{domain} -> {domain_age_in_days, mx, mx_records,+ *+ * P4-S3 (2026-10-27) reliability fixes (was 29/29 "unavailable" on the admin+ * health panel):+ * 1. The bare host `mailcheck.ai/domain/...` now 301s to usercheck.com+ * (Cloudflare JS challenge — unusable server-side). The keyless API host+ * `api.mailcheck.ai/domain/...` still serves the same JSON schema.+ * 2. api.mailcheck.ai enforces a tight unauthenticated per-IP quota+ * (observed: exhausted after ~5 requests/hour, Retry-After 3180s). Two+ * guards make that survivable:+ * - successful responses are cached per-domain for 24h (domain mail+ * posture is slow-changing; popular domains amortize to ~1 call/day);+ * - on API failure the last-good payload is served for up to 7 days+ * with an honest "(cached …)" marker — the stale-data grace period.+ * The card only degrades to unavailable when there has never been a+ * good response for that domain.+ *+ * GET https://api.mailcheck.ai/domain/{domain} -> {domain_age_in_days, mx, mx_records,* mx_providers, spf, dmarc, disposable, public_domain, relay_domain,* free_subdomain, spam, did_you_mean}*+ * NOTE v2 schema: spf/dmarc are strings ("strict"/"relaxed"/"none") or null+ * rather than booleans. parse() normalizes both shapes.+ ** Used for both email checks (deliverability + disposable detection) and* domain checks (mail hardening posture at a glance).*/class MailcheckSource implements OsintSource{- private const URL = 'https://mailcheck.ai/domain/';+ private const URL = 'https://api.mailcheck.ai/domain/';++ /** Fresh-response cache per domain (upstream is slow-changing data). */+ private const FRESH_TTL = 86400; // 24 h++ /** Last-good stale-grace window when the API is rate-limited/down. */+ private const STALE_TTL = 604800; // 7 dayspublic function key(): string{@@ -41,9 +67,7 @@ class MailcheckSource implements OsintSourcepublic function requests(string $type, string $identifier): array{- $domain = $type === 'email'- ? strtolower(substr(strrchr($identifier, '@') ?: '', 1))- : strtolower($identifier);+ $domain = $this->domainOf($type, $identifier);if ($domain === '' || !filter_var($domain, FILTER_VALIDATE_DOMAIN)) {return [];}@@ -60,22 +84,62 @@ class MailcheckSource implements OsintSourcepublic function parse(string $type, string $identifier, array $responses): array{+ $domain = $this->domainOf($type, $identifier);+ $cacheKey = $domain !== '' ? 'osint.src.mailcheck.v1:' . $domain : '';+$resp = $responses[$this->key() . '.main'] ?? null;- if (!$resp instanceof Response || !$resp->ok()) {- return ['status' => 'unavailable', 'summary' => 'Email domain intelligence unavailable right now.', 'data' => []];+ if ($resp instanceof Response && $resp->ok()) {+ $j = $resp->json();+ if (is_array($j) && $j !== []) {+ // Remember the last-good payload (fresh for 24h, then it only+ // serves the stale grace window until a newer success lands).+ if ($cacheKey !== '') {+ try {+ Cache::put($cacheKey, ['j' => $j, 'at' => time()], self::STALE_TTL);+ } catch (\Throwable $e) {+ // cache backend hiccup must never break the card+ }+ }++ return $this->verdict($j, false);+ }}- $j = $resp->json();- if (!is_array($j)) {- return ['status' => 'unavailable', 'summary' => 'Email domain intelligence returned an unexpected response.', 'data' => []];+ // API failed (rate-limit 429 / timeout / 5xx): stale-data grace period.+ if ($cacheKey !== '') {+ $cached = Cache::get($cacheKey);+ if (is_array($cached) && is_array($cached['j'] ?? null)) {+ $ageH = isset($cached['at']) ? (int) round((time() - (int) $cached['at']) / 3600) : null;++ return $this->verdict($cached['j'], true, $ageH);+ }}+ return ['status' => 'unavailable', 'summary' => 'Email domain intelligence unavailable right now.', 'data' => []];+ }++ /**+ * Build the card verdict from a mailcheck payload. $stale marks data+ * served from the last-good cache because the API call failed.+ *+ * @param array<string, mixed> $j+ */+ private function verdict(array $j, bool $stale, ?int $ageHours = null): array+ {$disposable = (bool) ($j['disposable'] ?? false);$mx = (bool) ($j['mx'] ?? false);$spam = (bool) ($j['spam'] ?? false);$age = (int) ($j['domain_age_in_days'] ?? 0);$dym = (string) ($j['did_you_mean'] ?? '');+ // v2 schema (P4-S3): spf/dmarc can be bool (legacy), string+ // ("strict"/"relaxed"/"none") or null. A published, enforced policy is+ // any non-empty string other than "none" (or a literal true).+ $spfPolicy = $j['spf'] ?? null;+ $dmarcPolicy = $j['dmarc'] ?? null;+ $spf = is_bool($spfPolicy) ? $spfPolicy : (is_string($spfPolicy) && !in_array(strtolower($spfPolicy), ['none', ''], true));+ $dmarc = is_bool($dmarcPolicy) ? $dmarcPolicy : (is_string($dmarcPolicy) && !in_array(strtolower($dmarcPolicy), ['none', ''], true));+$flags = [];if ($disposable) {$flags[] = 'disposable/temporary email provider';@@ -92,18 +156,29 @@ class MailcheckSource implements OsintSource'mx_present' => $mx,'spam_flagged' => $spam,'domain_age_days' => $age,- 'spf' => (bool) ($j['spf'] ?? false),- 'dmarc' => (bool) ($j['dmarc'] ?? false),+ 'spf' => $spf,+ 'dmarc' => $dmarc,'mx_providers' => is_array($j['mx_providers'] ?? null) ? array_slice($j['mx_providers'], 0, 4) : [],];+ // v2: keep the raw policy strings for reports ("" when unpublished).+ $data['spf_policy'] = is_string($spfPolicy) ? $spfPolicy : ($spf ? 'published' : '');+ $data['dmarc_policy'] = is_string($dmarcPolicy) ? $dmarcPolicy : ($dmarc ? 'published' : '');if ($dym !== '') {$data['did_you_mean'] = $dym;}+ if ($stale) {+ $data['cached'] = true;+ }++ $staleNote = '';+ if ($stale) {+ $staleNote = ' (cached' . ($ageHours !== null && $ageHours >= 0 ? ' ' . $ageHours . 'h old' : '') . ' — live API rate-limited)';+ }if ($flags !== []) {return ['status' => 'hit',- 'summary' => 'Email domain concerns: ' . implode('; ', $flags) . '.',+ 'summary' => 'Email domain concerns: ' . implode('; ', $flags) . '.' . $staleNote,'severity' => $disposable ? 'medium' : 'low','data' => $data,];@@ -113,9 +188,18 @@ class MailcheckSource implements OsintSource'status' => 'clean','summary' => 'Email domain looks healthy'. ($age > 0 ? ' (domain age ' . number_format($age) . ' days)' : '')- . ($data['spf'] && $data['dmarc'] ? ' with SPF+DMARC published' : '') . '.',+ . ($spf && $dmarc ? ' with SPF+DMARC published' : '') . '.' . $staleNote,'severity' => 'info','data' => $data,];}++ private function domainOf(string $type, string $identifier): string+ {+ $domain = $type === 'email'+ ? strtolower(substr(strrchr($identifier, '@') ?: '', 1))+ : strtolower($identifier);++ return $domain === '' ? '' : $domain;+ }}