Skip to content

Full Toolkit for Next-Level Domain Analysis

Bound subdomain discovery and preserve partial results 🕸️🪤 (#138)

Felix Wotschofsky8300ca7136cc
diff --git a/app/lookup/[domain]/subdomains/page.tsx b/app/lookup/[domain]/subdomains/page.tsx
index 7e8db93..5589d87 100644
--- a/app/lookup/[domain]/subdomains/page.tsx
+++ b/app/lookup/[domain]/subdomains/page.tsx
@@ -5,7 +5,6 @@ import type { FC } from 'react';
import { Button } from '@/components/ui/button';
import { generateCsv } from '@/lib/csv';
-import { CloudflareDoHResolver } from '@/lib/resolvers/cloudflare';
import { recordLookupAfter } from '@/lib/search';
import { findSubdomains } from '@/lib/subdomains';
import { getSourceLabel } from '@/lib/subfinder-sources';
@@ -35,13 +34,15 @@ export const generateMetadata = async ({
};
};
+export const maxDuration = 60;
+
const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({
params,
}) => {
const { domain } = await params;
- const { results, detailsReduced, detailedResultsLimit } =
- await findSubdomains(domain, new CloudflareDoHResolver());
+ const { results, discoveryIncomplete, detailsReduced, detailedResultsLimit } =
+ await findSubdomains(domain);
await recordLookupAfter(domain, 'subdomains', results.length > 0);
@@ -66,7 +67,9 @@ const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({
if (!results.length) {
return (
<p className="mt-8 text-center text-zinc-500 dark:text-zinc-400">
- Could not find any subdomains for this domain.
+ {discoveryIncomplete
+ ? 'Subdomain discovery timed out before finding any results. Please try again.'
+ : 'Could not find any subdomains for this domain.'}
</p>
);
}
@@ -100,8 +103,10 @@ const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({
<p className="mt-8 text-center text-zinc-500 dark:text-zinc-400">
Found {results.length} subdomains.
+ {discoveryIncomplete &&
+ ' Discovery timed out; these results are incomplete.'}
{detailsReduced &&
- ` Detailed results limited to ${detailedResultsLimit} subdomains. Some entries may lack information.`}
+ ' Some DNS checks could not be completed; their status is unknown.'}
</p>
</>
);
diff --git a/lib/subdomains.test.ts b/lib/subdomains.test.ts
new file mode 100644
index 0000000..19aefde
--- /dev/null
+++ b/lib/subdomains.test.ts
@@ -0,0 +1,138 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { findSubdomains } from './subdomains';
+
+const { execa } = vi.hoisted(() => ({ execa: vi.fn() }));
+vi.mock('execa', () => ({ execa }));
+
+const encodeDiscovery = (host: string) =>
+ JSON.stringify({ host, sources: ['test'] });
+
+describe('subdomain lookup budgets', () => {
+ beforeEach(() => vi.useFakeTimers());
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.clearAllMocks();
+ });
+
+ it('keeps complete JSON discoveries and drops the truncated line when subfinder is killed at its deadline', async () => {
+ execa.mockRejectedValue(
+ Object.assign(new Error('timeout'), {
+ timedOut: true,
+ stdout: `${encodeDiscovery('a.example.com')}\n{"host":"unfinished`,
+ }),
+ );
+
+ const resolveAnswers = vi
+ .fn()
+ .mockResolvedValue({ answers: [{ type: 1 }] });
+
+ const result = await findSubdomains('example.com', () => ({
+ resolveAnswers,
+ }));
+
+ expect(result.discoveryIncomplete).toBe(true);
+ expect(result.results).toEqual([
+ { domain: 'a.example.com', sources: ['test'], stillExists: true },
+ ]);
+ expect(execa.mock.calls[0][2]).toMatchObject({
+ timeout: 35_000,
+ killSignal: 'SIGKILL',
+ });
+ });
+
+ it('recovers source-labelled stderr hosts when subfinder is killed before flushing JSON', async () => {
+ execa.mockRejectedValue(
+ Object.assign(new Error('timeout'), {
+ timedOut: true,
+ stdout: '',
+ stderr:
+ '[INF] Enumerating subdomains for example.com\n[crtsh] a.example.com\n[alienvault] a.example.com\n[crtsh] a.example.com\n[ERR] b.example.com\n[crtsh] unrelated.net\n[crtsh] invalid/path\n',
+ }),
+ );
+
+ const resolveAnswers = vi.fn().mockResolvedValue({ answers: [] });
+ const result = await findSubdomains('example.com', () => ({
+ resolveAnswers,
+ }));
+
+ expect(result.discoveryIncomplete).toBe(true);
+ expect(result.results).toEqual([
+ {
+ domain: 'a.example.com',
+ sources: ['alienvault', 'crtsh'],
+ stillExists: false,
+ },
+ ]);
+ expect(execa.mock.calls[0][1]).toEqual(
+ expect.arrayContaining(['-v', '-nc', '-duc']),
+ );
+ expect(execa.mock.calls[0][1]).not.toContain('-silent');
+ });
+
+ it('cancels stalled DNS checks at the deadline, runs at most 10 at once, and keeps every discovery', async () => {
+ execa.mockResolvedValue({
+ stdout: Array.from({ length: 30 }, (_, i) =>
+ encodeDiscovery(`a${i}.example.com`),
+ ).join('\n'),
+ });
+
+ let signal!: AbortSignal;
+ const resolveAnswers = vi.fn(
+ () =>
+ new Promise<never>((_, reject) => {
+ signal.addEventListener('abort', () => reject(signal.reason), {
+ once: true,
+ });
+ }),
+ );
+
+ const pending = findSubdomains('example.com', (value) => {
+ signal = value;
+ return { resolveAnswers };
+ });
+
+ await vi.advanceTimersByTimeAsync(5_000);
+ const result = await pending;
+
+ expect(signal.aborted).toBe(true);
+ expect(resolveAnswers).toHaveBeenCalledTimes(10);
+ expect(result.results).toHaveLength(30);
+ expect(result.results.every((item) => item.stillExists === null)).toBe(
+ true,
+ );
+ expect(result.detailsReduced).toBe(true);
+ });
+
+ it('an empty DNS answer means the name is gone, a failed check means unknown', async () => {
+ execa.mockResolvedValue({
+ stdout: [
+ encodeDiscovery('a.example.com'),
+ encodeDiscovery('b.example.com'),
+ ].join('\n'),
+ });
+
+ const resolveAnswers = vi
+ .fn()
+ .mockResolvedValueOnce({ answers: [] })
+ .mockRejectedValueOnce(new Error('SERVFAIL'));
+
+ const result = await findSubdomains('example.com', () => ({
+ resolveAnswers,
+ }));
+
+ expect(result.results.map((item) => item.stillExists)).toEqual([
+ false,
+ null,
+ ]);
+ });
+
+ it('a subfinder crash surfaces as an error, not an empty discovery', async () => {
+ execa.mockRejectedValue(new Error('cannot execute'));
+
+ await expect(findSubdomains('example.com')).rejects.toThrow(
+ 'Subdomain discovery failed',
+ );
+ });
+});
diff --git a/lib/subdomains.ts b/lib/subdomains.ts
index 2fb2e03..68a19be 100644
--- a/lib/subdomains.ts
+++ b/lib/subdomains.ts
@@ -1,14 +1,18 @@
import path from 'node:path';
+import { log } from 'evlog';
import { execa } from 'execa';
import { z } from 'zod';
-import type { DnsResolver } from './resolvers/base';
+import { CloudflareDoHResolver } from './resolvers/cloudflare';
import { UserFacingError } from './user-facing-error';
import { isValidDomain } from './utils';
const DETAILED_RESULTS_LIMIT = 500;
const SUBFINDER_TIMEOUT_SECONDS = 8;
+const DISCOVERY_DEADLINE_MS = 35_000;
+const DETAILS_DEADLINE_MS = 5_000;
+const DETAILS_CONCURRENCY = 10;
const SUBFINDER_BIN = path.join(process.cwd(), 'bin', 'subfinder');
@@ -45,7 +49,9 @@ const runSubfinder = async (domain: string) => {
domain,
'-json',
'-cs', // collect-sources: emit one record per host with all sources
- '-silent',
+ '-v', // Source-labelled discoveries arrive before the final JSON output.
+ '-nc',
+ '-duc',
'-timeout',
String(SUBFINDER_TIMEOUT_SECONDS),
],
@@ -53,14 +59,31 @@ const runSubfinder = async (domain: string) => {
// Vercel's serverless filesystem is read-only except for /tmp; point
// subfinder's config dir there so first-run bootstrap can write.
env: { HOME: '/tmp' },
+ timeout: DISCOVERY_DEADLINE_MS,
+ killSignal: 'SIGKILL',
},
);
- return stdout
- .split('\n')
- .map(readDiscovery)
- .filter((discovery): discovery is Discovery => discovery !== null);
+ return { discoveries: parseDiscoveries(stdout), incomplete: false };
} catch (error) {
+ if (
+ error instanceof Error &&
+ 'timedOut' in error &&
+ error.timedOut &&
+ 'stdout' in error &&
+ typeof error.stdout === 'string'
+ ) {
+ return {
+ discoveries: recoverDiscoveries(
+ error.stdout,
+ 'stderr' in error && typeof error.stderr === 'string'
+ ? error.stderr
+ : '',
+ ),
+ incomplete: true,
+ };
+ }
+
throw new UserFacingError(
{
title: 'Subdomain discovery failed',
@@ -73,33 +96,115 @@ const runSubfinder = async (domain: string) => {
}
};
-export const findSubdomains = async (domain: string, resolver: DnsResolver) => {
- const discoveries = (await runSubfinder(domain))
+const parseDiscoveries = (stdout: string) =>
+ stdout
+ .split('\n')
+ .map(readDiscovery)
+ .filter((discovery): discovery is Discovery => discovery !== null);
+
+// subfinder v2.17.0 buffers JSON until enumeration ends but streams `[source] host` to stderr;
+// without these lines a timeout SIGKILL discards every unflushed discovery.
+// Format pinned with the version in scripts/install-subfinder.sh.
+const recoverDiscoveries = (stdout: string, stderr: string): Discovery[] => {
+ const discovered = new Map<string, Set<string>>();
+ const add = ({ host, sources }: Discovery) => {
+ const known = discovered.get(host) ?? new Set<string>();
+ sources.forEach((source) => known.add(source));
+ discovered.set(host, known);
+ };
+
+ parseDiscoveries(stdout).forEach(add);
+
+ for (const line of stderr.split('\n')) {
+ const match = /^\[([a-z][a-z0-9_-]*)\]\s+(\S+)\s*$/i.exec(line);
+ if (!match || /^(INF|DBG|WRN|ERR|FTL|VRB)$/i.test(match[1])) continue;
+ const host = denormalizeWildcard(match[2].toLowerCase().replace(/\.$/, ''));
+ if (isValidDomain(host)) add({ host, sources: [match[1].toLowerCase()] });
+ }
+
+ return [...discovered].map(([host, sources]) => ({
+ host,
+ sources: [...sources],
+ }));
+};
+
+type CreateResolver = (
+ signal: AbortSignal,
+) => Pick<CloudflareDoHResolver, 'resolveAnswers'>;
+
+export const findSubdomains = async (
+ domain: string,
+ createResolver: CreateResolver = (signal) =>
+ new CloudflareDoHResolver({ signal }),
+) => {
+ const startedAt = Date.now();
+ const discovery = await runSubfinder(domain);
+ const discoveryMs = Date.now() - startedAt;
+ const results = discovery.discoveries
.filter(({ host }) => isValidDomain(host) && host.endsWith(`.${domain}`))
.map(({ host, sources }) => ({
domain: host,
sources: sources.toSorted(),
+ stillExists: null as boolean | null,
}))
.toSorted((a, b) => a.domain.localeCompare(b.domain));
- const results = await Promise.all(
- discoveries.map(async (discovery, index) => {
- // Limited to avoid subrequest limits in serverless functions.
- if (index >= DETAILED_RESULTS_LIMIT) {
- return { ...discovery, stillExists: null };
- }
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), DETAILS_DEADLINE_MS);
+ const resolver = createResolver(controller.signal);
+ let nextIndex = 0;
- const records = await resolver.resolveRecordType(discovery.domain, 'A');
- return {
- ...discovery,
- stillExists: records.records.length > 0,
- };
- }),
- );
+ try {
+ await Promise.all(
+ Array.from(
+ { length: Math.min(DETAILS_CONCURRENCY, results.length) },
+ async () => {
+ while (
+ !controller.signal.aborted &&
+ nextIndex < Math.min(results.length, DETAILED_RESULTS_LIMIT)
+ ) {
+ const result = results[nextIndex++];
+
+ try {
+ const response = await resolver.resolveAnswers(
+ result.domain,
+ 'A',
+ );
+
+ if (!controller.signal.aborted)
+ result.stillExists = response.answers.some(
+ (answer) => answer.type === 1,
+ );
+ } catch {
+ // A failed DNS check says nothing about whether the discovered name exists.
+ }
+ }
+ },
+ ),
+ );
+ } finally {
+ clearTimeout(timer);
+ controller.abort();
+ }
+
+ const uncheckedCount = results.filter(
+ (result) => result.stillExists === null,
+ ).length;
+
+ log.info({
+ message: 'subdomains_lookup_completed',
+ domain,
+ discoveryMs,
+ durationMs: Date.now() - startedAt,
+ resultCount: results.length,
+ uncheckedCount,
+ incomplete: discovery.incomplete,
+ });
return {
results,
- detailsReduced: discoveries.length > DETAILED_RESULTS_LIMIT,
+ discoveryIncomplete: discovery.incomplete,
+ detailsReduced: uncheckedCount > 0,
detailedResultsLimit: DETAILED_RESULTS_LIMIT,
};
};