Skip to content

Self-hosted OSINT exposure checker — is your email, username, phone or password leaked? Keyless sources: breaches, infostealer logs, COMB credentials, HIBP k-anonymity, 200-platform username grid. Docker one-command install, no API keys, no AI, Apache-2.0.

sync(ec): crt.sh: Cert Spotter fallback; mailcheck: api.mailcheck.ai host + 24h cache + stale grace (P4-S3 fixes)

Rana Khan61f286e06625
diff --git a/app/Modules/Osint/Sources/CrtShSource.php b/app/Modules/Osint/Sources/CrtShSource.php
index 31e8dd0..fd32a9b 100644
--- a/app/Modules/Osint/Sources/CrtShSource.php
+++ b/app/Modules/Osint/Sources/CrtShSource.php
@@ -7,14 +7,28 @@ namespace App\Modules\Osint\Sources;
use Illuminate\Http\Client\Response;
/**
- * crt.sh certificate-transparency subdomain discovery (keyless).
- * GET https://crt.sh/?q=%.{domain}&output=json&limit=200 — every subdomain
- * that ever obtained a TLS certificate. Counts + names of the first 40.
- * crt.sh can be slow; 8s timeout, graceful unavailable on failure.
+ * Certificate-transparency subdomain discovery (keyless), P4-S3 hardened.
+ *
+ * crt.sh (`GET https://crt.sh/?q=%.{domain}&output=json&limit=200`) is the
+ * classic keyless CT source but is frequently slow (>9s pool budget) or
+ * returns 502 under load — 1-in-4 domain checks degraded to "unavailable"
+ * and the admin health panel showed the tool rose.
+ *
+ * P4-S3 (2026-10-27) dual-endpoint strategy: BOTH requests are fired in the
+ * same parallel Http::pool (zero added latency) and parse() uses whichever
+ * answered:
+ * crtsh.main — crt.sh JSON API (primary, unlimited but slow/flaky)
+ * crtsh.alt — Cert Spotter public issuances API (fast 200ms, keyless,
+ * 100 req/hour unauthenticated — fine at our volume, and the
+ * 30-min per-identifier cache absorbs repeats)
+ *
+ * Output shape is unchanged (subdomains + subdomain_count), so the exposure
+ * check UI and Trace reports keep working.
*/
class CrtShSource implements OsintSource
{
- private const URL = 'https://crt.sh/?q=';
+ private const CRTSH_URL = 'https://crt.sh/?q=';
+ private const CERTSPOTTER_URL = 'https://api.certspotter.com/v1/issuances';
public function key(): string
{
@@ -38,42 +52,125 @@ class CrtShSource implements OsintSource
public function requests(string $type, string $identifier): array
{
+ $domain = strtolower($identifier);
+
return [
[
'name' => $this->key() . '.main',
'method' => 'GET',
- 'url' => self::URL . urlencode('%.' . strtolower($identifier)) . '&output=json&limit=200',
- 'headers' => ['Accept' => 'application/json'],
+ 'url' => self::CRTSH_URL . urlencode('%.' . $domain) . '&output=json&limit=200',
+ 'headers' => [
+ 'Accept' => 'application/json',
+ // crt.sh intermittently 502s generic bot UAs; identify honestly.
+ 'User-Agent' => '2mny-trace/1.0 (+https://2mny.com)',
+ ],
+ ],
+ [
+ 'name' => $this->key() . '.alt',
+ 'method' => 'GET',
+ 'url' => self::CERTSPOTTER_URL . '?domain=' . urlencode($domain) . '&include_subdomains=true&expand=dns_names',
+ 'headers' => [
+ 'Accept' => 'application/json',
+ 'User-Agent' => '2mny-trace/1.0 (+https://2mny.com)',
+ ],
],
];
}
public function parse(string $type, string $identifier, array $responses): array
{
- $resp = $responses[$this->key() . '.main'] ?? null;
- if (!$resp instanceof Response || !$resp->ok()) {
- return ['status' => 'unavailable', 'summary' => 'Certificate Transparency lookup timed out (crt.sh is slow at times).', 'data' => []];
+ $identifier = strtolower($identifier);
+
+ // ---- Primary: crt.sh JSON -------------------------------------------
+ $main = $responses[$this->key() . '.main'] ?? null;
+ if ($main instanceof Response && $main->ok()) {
+ $j = $main->json();
+ if (is_array($j) && $j !== []) {
+ $names = $this->filterNames($this->namesFromCrtSh($j), $identifier);
+ if ($names !== []) {
+ return $this->result($names, 'crt.sh');
+ }
+ // Empty-but-valid crt.sh response: a legitimately unlisted
+ // domain — still check the alternate before calling it clean.
+ $alt = $this->parseAlt($responses, $identifier);
+ if ($alt !== null) {
+ return $alt;
+ }
+
+ return [
+ 'status' => 'clean',
+ 'summary' => 'No certificates found in Certificate Transparency logs for this domain.',
+ 'severity' => 'info',
+ 'data' => ['subdomains' => [], 'subdomain_count' => 0],
+ ];
+ }
}
- $j = $resp->json();
- if (!is_array($j)) {
- return ['status' => 'unavailable', 'summary' => 'Certificate Transparency returned an unexpected response.', 'data' => []];
+ // ---- Fallback: Cert Spotter issuances -------------------------------
+ $alt = $this->parseAlt($responses, $identifier);
+ if ($alt !== null) {
+ return $alt;
}
+ return ['status' => 'unavailable', 'summary' => 'Certificate Transparency lookup timed out (crt.sh is slow at times).', 'data' => []];
+ }
+
+ /**
+ * @param array<int, mixed> $rows
+ * @return array<int, string>
+ */
+ private function namesFromCrtSh(array $rows): array
+ {
$names = [];
- foreach ($j as $row) {
+ foreach ($rows as $row) {
+ if (!is_array($row)) {
+ continue;
+ }
$name = strtolower((string) ($row['name_value'] ?? ''));
foreach (explode("\n", $name) as $n) {
$n = trim($n, " *.\"\t");
- if ($n !== '' && str_contains($n, $identifier)) {
+ if ($n !== '') {
+ $names[$n] = true;
+ }
+ }
+ }
+
+ return array_keys($names);
+ }
+
+ /**
+ * Parse the Cert Spotter alternate response; null when it is unusable
+ * (missing, failed, rate-limited or unparseable) so the caller can fall
+ * through to honest-unavailable.
+ */
+ private function parseAlt(array $responses, string $identifier): ?array
+ {
+ $alt = $responses[$this->key() . '.alt'] ?? null;
+ if (!$alt instanceof Response || !$alt->ok()) {
+ return null;
+ }
+
+ $j = $alt->json();
+ if (!is_array($j)) {
+ return null;
+ }
+
+ $names = [];
+ foreach ($j as $row) {
+ if (!is_array($row) || !is_array($row['dns_names'] ?? null)) {
+ continue;
+ }
+ foreach ($row['dns_names'] as $n) {
+ $n = strtolower(trim((string) $n, " *.\"\t"));
+ if ($n !== '') {
$names[$n] = true;
}
}
}
- $names = array_keys($names);
- sort($names);
+ $names = $this->filterNames(array_keys($names), $identifier);
if ($names === []) {
+ // Valid response, no in-domain names: honest clean.
return [
'status' => 'clean',
'summary' => 'No certificates found in Certificate Transparency logs for this domain.',
@@ -82,6 +179,35 @@ class CrtShSource implements OsintSource
];
}
+ return $this->result($names, 'Cert Spotter CT mirror');
+ }
+
+ /**
+ * Keep only names that belong to the queried domain (both endpoints can
+ * return sibling noise), deduplicated + sorted.
+ *
+ * @param array<int, string> $names
+ * @return array<int, string>
+ */
+ private function filterNames(array $names, string $identifier): array
+ {
+ $out = [];
+ foreach ($names as $n) {
+ if ($n === $identifier || str_ends_with($n, '.' . $identifier)) {
+ $out[$n] = true;
+ }
+ }
+ $out = array_keys($out);
+ sort($out);
+
+ return $out;
+ }
+
+ /**
+ * @param array<int, string> $names
+ */
+ private function result(array $names, string $via): array
+ {
return [
'status' => 'hit',
'summary' => count($names) . ' hostname(s) found in Certificate Transparency logs — this is the domain\'s public TLS footprint.',
@@ -89,6 +215,7 @@ class CrtShSource implements OsintSource
'data' => [
'subdomain_count' => count($names),
'subdomains' => array_slice($names, 0, 40),
+ 'source' => $via,
],
];
}
diff --git a/app/Modules/Osint/Sources/MailcheckSource.php b/app/Modules/Osint/Sources/MailcheckSource.php
index b2279fb..4f61a21 100644
--- a/app/Modules/Osint/Sources/MailcheckSource.php
+++ b/app/Modules/Osint/Sources/MailcheckSource.php
@@ -5,19 +5,45 @@ declare(strict_types=1);
namespace App\Modules\Osint\Sources;
use Illuminate\Http\Client\Response;
+use Illuminate\Support\Facades\Cache;
/**
* mailcheck.ai - keyless email-domain intelligence.
- * GET https://mailcheck.ai/domain/{domain} -> {domain_age_in_days, mx, mx_records,
+ *
+ * P4-S3 (2026-10-27) reliability fixes (was 29/29 "unavailable" on the admin
+ * health panel):
+ * 1. The bare host `mailcheck.ai/domain/...` now 301s to usercheck.com
+ * (Cloudflare JS challenge — unusable server-side). The keyless API host
+ * `api.mailcheck.ai/domain/...` still serves the same JSON schema.
+ * 2. api.mailcheck.ai enforces a tight unauthenticated per-IP quota
+ * (observed: exhausted after ~5 requests/hour, Retry-After 3180s). Two
+ * guards make that survivable:
+ * - successful responses are cached per-domain for 24h (domain mail
+ * posture is slow-changing; popular domains amortize to ~1 call/day);
+ * - on API failure the last-good payload is served for up to 7 days
+ * with an honest "(cached …)" marker — the stale-data grace period.
+ * The card only degrades to unavailable when there has never been a
+ * good response for that domain.
+ *
+ * GET https://api.mailcheck.ai/domain/{domain} -> {domain_age_in_days, mx, mx_records,
* mx_providers, spf, dmarc, disposable, public_domain, relay_domain,
* free_subdomain, spam, did_you_mean}
*
+ * NOTE v2 schema: spf/dmarc are strings ("strict"/"relaxed"/"none") or null
+ * rather than booleans. parse() normalizes both shapes.
+ *
* Used for both email checks (deliverability + disposable detection) and
* domain checks (mail hardening posture at a glance).
*/
class MailcheckSource implements OsintSource
{
- private const URL = 'https://mailcheck.ai/domain/';
+ private const URL = 'https://api.mailcheck.ai/domain/';
+
+ /** Fresh-response cache per domain (upstream is slow-changing data). */
+ private const FRESH_TTL = 86400; // 24 h
+
+ /** Last-good stale-grace window when the API is rate-limited/down. */
+ private const STALE_TTL = 604800; // 7 days
public function key(): string
{
@@ -41,9 +67,7 @@ class MailcheckSource implements OsintSource
public function requests(string $type, string $identifier): array
{
- $domain = $type === 'email'
- ? strtolower(substr(strrchr($identifier, '@') ?: '', 1))
- : strtolower($identifier);
+ $domain = $this->domainOf($type, $identifier);
if ($domain === '' || !filter_var($domain, FILTER_VALIDATE_DOMAIN)) {
return [];
}
@@ -60,22 +84,62 @@ class MailcheckSource implements OsintSource
public function parse(string $type, string $identifier, array $responses): array
{
+ $domain = $this->domainOf($type, $identifier);
+ $cacheKey = $domain !== '' ? 'osint.src.mailcheck.v1:' . $domain : '';
+
$resp = $responses[$this->key() . '.main'] ?? null;
- if (!$resp instanceof Response || !$resp->ok()) {
- return ['status' => 'unavailable', 'summary' => 'Email domain intelligence unavailable right now.', 'data' => []];
+ if ($resp instanceof Response && $resp->ok()) {
+ $j = $resp->json();
+ if (is_array($j) && $j !== []) {
+ // Remember the last-good payload (fresh for 24h, then it only
+ // serves the stale grace window until a newer success lands).
+ if ($cacheKey !== '') {
+ try {
+ Cache::put($cacheKey, ['j' => $j, 'at' => time()], self::STALE_TTL);
+ } catch (\Throwable $e) {
+ // cache backend hiccup must never break the card
+ }
+ }
+
+ return $this->verdict($j, false);
+ }
}
- $j = $resp->json();
- if (!is_array($j)) {
- return ['status' => 'unavailable', 'summary' => 'Email domain intelligence returned an unexpected response.', 'data' => []];
+ // API failed (rate-limit 429 / timeout / 5xx): stale-data grace period.
+ if ($cacheKey !== '') {
+ $cached = Cache::get($cacheKey);
+ if (is_array($cached) && is_array($cached['j'] ?? null)) {
+ $ageH = isset($cached['at']) ? (int) round((time() - (int) $cached['at']) / 3600) : null;
+
+ return $this->verdict($cached['j'], true, $ageH);
+ }
}
+ return ['status' => 'unavailable', 'summary' => 'Email domain intelligence unavailable right now.', 'data' => []];
+ }
+
+ /**
+ * Build the card verdict from a mailcheck payload. $stale marks data
+ * served from the last-good cache because the API call failed.
+ *
+ * @param array<string, mixed> $j
+ */
+ private function verdict(array $j, bool $stale, ?int $ageHours = null): array
+ {
$disposable = (bool) ($j['disposable'] ?? false);
$mx = (bool) ($j['mx'] ?? false);
$spam = (bool) ($j['spam'] ?? false);
$age = (int) ($j['domain_age_in_days'] ?? 0);
$dym = (string) ($j['did_you_mean'] ?? '');
+ // v2 schema (P4-S3): spf/dmarc can be bool (legacy), string
+ // ("strict"/"relaxed"/"none") or null. A published, enforced policy is
+ // any non-empty string other than "none" (or a literal true).
+ $spfPolicy = $j['spf'] ?? null;
+ $dmarcPolicy = $j['dmarc'] ?? null;
+ $spf = is_bool($spfPolicy) ? $spfPolicy : (is_string($spfPolicy) && !in_array(strtolower($spfPolicy), ['none', ''], true));
+ $dmarc = is_bool($dmarcPolicy) ? $dmarcPolicy : (is_string($dmarcPolicy) && !in_array(strtolower($dmarcPolicy), ['none', ''], true));
+
$flags = [];
if ($disposable) {
$flags[] = 'disposable/temporary email provider';
@@ -92,18 +156,29 @@ class MailcheckSource implements OsintSource
'mx_present' => $mx,
'spam_flagged' => $spam,
'domain_age_days' => $age,
- 'spf' => (bool) ($j['spf'] ?? false),
- 'dmarc' => (bool) ($j['dmarc'] ?? false),
+ 'spf' => $spf,
+ 'dmarc' => $dmarc,
'mx_providers' => is_array($j['mx_providers'] ?? null) ? array_slice($j['mx_providers'], 0, 4) : [],
];
+ // v2: keep the raw policy strings for reports ("" when unpublished).
+ $data['spf_policy'] = is_string($spfPolicy) ? $spfPolicy : ($spf ? 'published' : '');
+ $data['dmarc_policy'] = is_string($dmarcPolicy) ? $dmarcPolicy : ($dmarc ? 'published' : '');
if ($dym !== '') {
$data['did_you_mean'] = $dym;
}
+ if ($stale) {
+ $data['cached'] = true;
+ }
+
+ $staleNote = '';
+ if ($stale) {
+ $staleNote = ' (cached' . ($ageHours !== null && $ageHours >= 0 ? ' ' . $ageHours . 'h old' : '') . ' — live API rate-limited)';
+ }
if ($flags !== []) {
return [
'status' => 'hit',
- 'summary' => 'Email domain concerns: ' . implode('; ', $flags) . '.',
+ 'summary' => 'Email domain concerns: ' . implode('; ', $flags) . '.' . $staleNote,
'severity' => $disposable ? 'medium' : 'low',
'data' => $data,
];
@@ -113,9 +188,18 @@ class MailcheckSource implements OsintSource
'status' => 'clean',
'summary' => 'Email domain looks healthy'
. ($age > 0 ? ' (domain age ' . number_format($age) . ' days)' : '')
- . ($data['spf'] && $data['dmarc'] ? ' with SPF+DMARC published' : '') . '.',
+ . ($spf && $dmarc ? ' with SPF+DMARC published' : '') . '.' . $staleNote,
'severity' => 'info',
'data' => $data,
];
}
+
+ private function domainOf(string $type, string $identifier): string
+ {
+ $domain = $type === 'email'
+ ? strtolower(substr(strrchr($identifier, '@') ?: '', 1))
+ : strtolower($identifier);
+
+ return $domain === '' ? '' : $domain;
+ }
}