Full Toolkit for Next-Level Domain Analysis
Bound subdomain discovery and preserve partial results 🕸️🪤 (#138)
Felix Wotschofsky8300ca7136cc
diff --git a/app/lookup/[domain]/subdomains/page.tsx b/app/lookup/[domain]/subdomains/page.tsxindex 7e8db93..5589d87 100644--- a/app/lookup/[domain]/subdomains/page.tsx+++ b/app/lookup/[domain]/subdomains/page.tsx@@ -5,7 +5,6 @@ import type { FC } from 'react';import { Button } from '@/components/ui/button';import { generateCsv } from '@/lib/csv';-import { CloudflareDoHResolver } from '@/lib/resolvers/cloudflare';import { recordLookupAfter } from '@/lib/search';import { findSubdomains } from '@/lib/subdomains';import { getSourceLabel } from '@/lib/subfinder-sources';@@ -35,13 +34,15 @@ export const generateMetadata = async ({};};+export const maxDuration = 60;+const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({params,}) => {const { domain } = await params;- const { results, detailsReduced, detailedResultsLimit } =- await findSubdomains(domain, new CloudflareDoHResolver());+ const { results, discoveryIncomplete, detailsReduced, detailedResultsLimit } =+ await findSubdomains(domain);await recordLookupAfter(domain, 'subdomains', results.length > 0);@@ -66,7 +67,9 @@ const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({if (!results.length) {return (<p className="mt-8 text-center text-zinc-500 dark:text-zinc-400">- Could not find any subdomains for this domain.+ {discoveryIncomplete+ ? 'Subdomain discovery timed out before finding any results. Please try again.'+ : 'Could not find any subdomains for this domain.'}</p>);}@@ -100,8 +103,10 @@ const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({<p className="mt-8 text-center text-zinc-500 dark:text-zinc-400">Found {results.length} subdomains.+ {discoveryIncomplete &&+ ' Discovery timed out; these results are incomplete.'}{detailsReduced &&- ` Detailed results limited to ${detailedResultsLimit} subdomains. Some entries may lack information.`}+ ' Some DNS checks could not be completed; their status is unknown.'}</p></>);diff --git a/lib/subdomains.test.ts b/lib/subdomains.test.tsnew file mode 100644index 0000000..19aefde--- /dev/null+++ b/lib/subdomains.test.ts@@ -0,0 +1,138 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';++import { findSubdomains } from './subdomains';++const { execa } = vi.hoisted(() => ({ execa: vi.fn() }));+vi.mock('execa', () => ({ execa }));++const encodeDiscovery = (host: string) =>+ JSON.stringify({ host, sources: ['test'] });++describe('subdomain lookup budgets', () => {+ beforeEach(() => vi.useFakeTimers());++ afterEach(() => {+ vi.useRealTimers();+ vi.clearAllMocks();+ });++ it('keeps complete JSON discoveries and drops the truncated line when subfinder is killed at its deadline', async () => {+ execa.mockRejectedValue(+ Object.assign(new Error('timeout'), {+ timedOut: true,+ stdout: `${encodeDiscovery('a.example.com')}\n{"host":"unfinished`,+ }),+ );++ const resolveAnswers = vi+ .fn()+ .mockResolvedValue({ answers: [{ type: 1 }] });++ const result = await findSubdomains('example.com', () => ({+ resolveAnswers,+ }));++ expect(result.discoveryIncomplete).toBe(true);+ expect(result.results).toEqual([+ { domain: 'a.example.com', sources: ['test'], stillExists: true },+ ]);+ expect(execa.mock.calls[0][2]).toMatchObject({+ timeout: 35_000,+ killSignal: 'SIGKILL',+ });+ });++ it('recovers source-labelled stderr hosts when subfinder is killed before flushing JSON', async () => {+ execa.mockRejectedValue(+ Object.assign(new Error('timeout'), {+ timedOut: true,+ stdout: '',+ stderr:+ '[INF] Enumerating subdomains for example.com\n[crtsh] a.example.com\n[alienvault] a.example.com\n[crtsh] a.example.com\n[ERR] b.example.com\n[crtsh] unrelated.net\n[crtsh] invalid/path\n',+ }),+ );++ const resolveAnswers = vi.fn().mockResolvedValue({ answers: [] });+ const result = await findSubdomains('example.com', () => ({+ resolveAnswers,+ }));++ expect(result.discoveryIncomplete).toBe(true);+ expect(result.results).toEqual([+ {+ domain: 'a.example.com',+ sources: ['alienvault', 'crtsh'],+ stillExists: false,+ },+ ]);+ expect(execa.mock.calls[0][1]).toEqual(+ expect.arrayContaining(['-v', '-nc', '-duc']),+ );+ expect(execa.mock.calls[0][1]).not.toContain('-silent');+ });++ it('cancels stalled DNS checks at the deadline, runs at most 10 at once, and keeps every discovery', async () => {+ execa.mockResolvedValue({+ stdout: Array.from({ length: 30 }, (_, i) =>+ encodeDiscovery(`a${i}.example.com`),+ ).join('\n'),+ });++ let signal!: AbortSignal;+ const resolveAnswers = vi.fn(+ () =>+ new Promise<never>((_, reject) => {+ signal.addEventListener('abort', () => reject(signal.reason), {+ once: true,+ });+ }),+ );++ const pending = findSubdomains('example.com', (value) => {+ signal = value;+ return { resolveAnswers };+ });++ await vi.advanceTimersByTimeAsync(5_000);+ const result = await pending;++ expect(signal.aborted).toBe(true);+ expect(resolveAnswers).toHaveBeenCalledTimes(10);+ expect(result.results).toHaveLength(30);+ expect(result.results.every((item) => item.stillExists === null)).toBe(+ true,+ );+ expect(result.detailsReduced).toBe(true);+ });++ it('an empty DNS answer means the name is gone, a failed check means unknown', async () => {+ execa.mockResolvedValue({+ stdout: [+ encodeDiscovery('a.example.com'),+ encodeDiscovery('b.example.com'),+ ].join('\n'),+ });++ const resolveAnswers = vi+ .fn()+ .mockResolvedValueOnce({ answers: [] })+ .mockRejectedValueOnce(new Error('SERVFAIL'));++ const result = await findSubdomains('example.com', () => ({+ resolveAnswers,+ }));++ expect(result.results.map((item) => item.stillExists)).toEqual([+ false,+ null,+ ]);+ });++ it('a subfinder crash surfaces as an error, not an empty discovery', async () => {+ execa.mockRejectedValue(new Error('cannot execute'));++ await expect(findSubdomains('example.com')).rejects.toThrow(+ 'Subdomain discovery failed',+ );+ });+});diff --git a/lib/subdomains.ts b/lib/subdomains.tsindex 2fb2e03..68a19be 100644--- a/lib/subdomains.ts+++ b/lib/subdomains.ts@@ -1,14 +1,18 @@import path from 'node:path';+import { log } from 'evlog';import { execa } from 'execa';import { z } from 'zod';-import type { DnsResolver } from './resolvers/base';+import { CloudflareDoHResolver } from './resolvers/cloudflare';import { UserFacingError } from './user-facing-error';import { isValidDomain } from './utils';const DETAILED_RESULTS_LIMIT = 500;const SUBFINDER_TIMEOUT_SECONDS = 8;+const DISCOVERY_DEADLINE_MS = 35_000;+const DETAILS_DEADLINE_MS = 5_000;+const DETAILS_CONCURRENCY = 10;const SUBFINDER_BIN = path.join(process.cwd(), 'bin', 'subfinder');@@ -45,7 +49,9 @@ const runSubfinder = async (domain: string) => {domain,'-json','-cs', // collect-sources: emit one record per host with all sources- '-silent',+ '-v', // Source-labelled discoveries arrive before the final JSON output.+ '-nc',+ '-duc','-timeout',String(SUBFINDER_TIMEOUT_SECONDS),],@@ -53,14 +59,31 @@ const runSubfinder = async (domain: string) => {// Vercel's serverless filesystem is read-only except for /tmp; point// subfinder's config dir there so first-run bootstrap can write.env: { HOME: '/tmp' },+ timeout: DISCOVERY_DEADLINE_MS,+ killSignal: 'SIGKILL',},);- return stdout- .split('\n')- .map(readDiscovery)- .filter((discovery): discovery is Discovery => discovery !== null);+ return { discoveries: parseDiscoveries(stdout), incomplete: false };} catch (error) {+ if (+ error instanceof Error &&+ 'timedOut' in error &&+ error.timedOut &&+ 'stdout' in error &&+ typeof error.stdout === 'string'+ ) {+ return {+ discoveries: recoverDiscoveries(+ error.stdout,+ 'stderr' in error && typeof error.stderr === 'string'+ ? error.stderr+ : '',+ ),+ incomplete: true,+ };+ }+throw new UserFacingError({title: 'Subdomain discovery failed',@@ -73,33 +96,115 @@ const runSubfinder = async (domain: string) => {}};-export const findSubdomains = async (domain: string, resolver: DnsResolver) => {- const discoveries = (await runSubfinder(domain))+const parseDiscoveries = (stdout: string) =>+ stdout+ .split('\n')+ .map(readDiscovery)+ .filter((discovery): discovery is Discovery => discovery !== null);++// subfinder v2.17.0 buffers JSON until enumeration ends but streams `[source] host` to stderr;+// without these lines a timeout SIGKILL discards every unflushed discovery.+// Format pinned with the version in scripts/install-subfinder.sh.+const recoverDiscoveries = (stdout: string, stderr: string): Discovery[] => {+ const discovered = new Map<string, Set<string>>();+ const add = ({ host, sources }: Discovery) => {+ const known = discovered.get(host) ?? new Set<string>();+ sources.forEach((source) => known.add(source));+ discovered.set(host, known);+ };++ parseDiscoveries(stdout).forEach(add);++ for (const line of stderr.split('\n')) {+ const match = /^\[([a-z][a-z0-9_-]*)\]\s+(\S+)\s*$/i.exec(line);+ if (!match || /^(INF|DBG|WRN|ERR|FTL|VRB)$/i.test(match[1])) continue;+ const host = denormalizeWildcard(match[2].toLowerCase().replace(/\.$/, ''));+ if (isValidDomain(host)) add({ host, sources: [match[1].toLowerCase()] });+ }++ return [...discovered].map(([host, sources]) => ({+ host,+ sources: [...sources],+ }));+};++type CreateResolver = (+ signal: AbortSignal,+) => Pick<CloudflareDoHResolver, 'resolveAnswers'>;++export const findSubdomains = async (+ domain: string,+ createResolver: CreateResolver = (signal) =>+ new CloudflareDoHResolver({ signal }),+) => {+ const startedAt = Date.now();+ const discovery = await runSubfinder(domain);+ const discoveryMs = Date.now() - startedAt;+ const results = discovery.discoveries.filter(({ host }) => isValidDomain(host) && host.endsWith(`.${domain}`)).map(({ host, sources }) => ({domain: host,sources: sources.toSorted(),+ stillExists: null as boolean | null,})).toSorted((a, b) => a.domain.localeCompare(b.domain));- const results = await Promise.all(- discoveries.map(async (discovery, index) => {- // Limited to avoid subrequest limits in serverless functions.- if (index >= DETAILED_RESULTS_LIMIT) {- return { ...discovery, stillExists: null };- }+ const controller = new AbortController();+ const timer = setTimeout(() => controller.abort(), DETAILS_DEADLINE_MS);+ const resolver = createResolver(controller.signal);+ let nextIndex = 0;- const records = await resolver.resolveRecordType(discovery.domain, 'A');- return {- ...discovery,- stillExists: records.records.length > 0,- };- }),- );+ try {+ await Promise.all(+ Array.from(+ { length: Math.min(DETAILS_CONCURRENCY, results.length) },+ async () => {+ while (+ !controller.signal.aborted &&+ nextIndex < Math.min(results.length, DETAILED_RESULTS_LIMIT)+ ) {+ const result = results[nextIndex++];++ try {+ const response = await resolver.resolveAnswers(+ result.domain,+ 'A',+ );++ if (!controller.signal.aborted)+ result.stillExists = response.answers.some(+ (answer) => answer.type === 1,+ );+ } catch {+ // A failed DNS check says nothing about whether the discovered name exists.+ }+ }+ },+ ),+ );+ } finally {+ clearTimeout(timer);+ controller.abort();+ }++ const uncheckedCount = results.filter(+ (result) => result.stillExists === null,+ ).length;++ log.info({+ message: 'subdomains_lookup_completed',+ domain,+ discoveryMs,+ durationMs: Date.now() - startedAt,+ resultCount: results.length,+ uncheckedCount,+ incomplete: discovery.incomplete,+ });return {results,- detailsReduced: discoveries.length > DETAILED_RESULTS_LIMIT,+ discoveryIncomplete: discovery.incomplete,+ detailsReduced: uncheckedCount > 0,detailedResultsLimit: DETAILED_RESULTS_LIMIT,};};