Full Toolkit for Next-Level Domain Analysis
More refactoring 🏗️
Felix Wotschofskyb4841b8ed802
diff --git a/app/lookup/[domain]/map/_components/BaseAlert.tsx b/app/lookup/[domain]/map/_components/BaseAlert.tsxnew file mode 100644index 0000000..064a001--- /dev/null+++ b/app/lookup/[domain]/map/_components/BaseAlert.tsx@@ -0,0 +1,20 @@+import type { LucideIcon } from 'lucide-react';+import type { FC } from 'react';++import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';++type BaseAlertProps = {+ icon: LucideIcon;+ title: string;+ children: React.ReactNode;+};++const BaseAlert: FC<BaseAlertProps> = ({ icon: Icon, title, children }) => (+ <Alert className="mx-auto mt-24 max-w-max">+ <Icon className="h-4 w-4" />+ <AlertTitle>{title}</AlertTitle>+ <AlertDescription>{children}</AlertDescription>+ </Alert>+);++export default BaseAlert;diff --git a/app/lookup/[domain]/map/page.tsx b/app/lookup/[domain]/map/page.tsxindex 520306b..c12c95a 100644--- a/app/lookup/[domain]/map/page.tsx+++ b/app/lookup/[domain]/map/page.tsx@@ -1,58 +1,19 @@-import { isbot } from 'isbot';import { CheckCircleIcon, InfoIcon, ShieldAlertIcon } from 'lucide-react';import type { Metadata } from 'next';import { headers } from 'next/headers';import type { FC } from 'react';-import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';-+import { isUserBot } from '@/lib/api';import { REGIONS } from '@/lib/data';import InternalDoHResolver from '@/lib/resolvers/InternalDoHResolver';+import BaseAlert from './_components/BaseAlert';import ResultsGlobe from './_components/ResultsGlobe';export const runtime = 'edge';-type MapResultsPageProps = {- params: {- domain: string;- };-};--export const generateMetadata = ({- params: { domain },-}: MapResultsPageProps): Metadata => ({- openGraph: {- url: `/lookup/${domain}/map`,- },- alternates: {- canonical: `/lookup/${domain}/map`,- },-});--const MapResultsPage: FC<MapResultsPageProps> = async ({- params: { domain },-}) => {- const userAgent = headers().get('user-agent');- const shouldBlockRequest = !userAgent || isbot(userAgent);-- if (shouldBlockRequest) {- console.log('Bot detected, blocking request, UA:', userAgent);- return (- <Alert className="mx-auto mt-24 max-w-max">- <ShieldAlertIcon className="h-4 w-4" />- <AlertTitle>Bot or crawler detected!</AlertTitle>- <AlertDescription>- To protect our infrastructure, this page is not available for bots or- crawlers.- <br />- But don't be sad, there's to crawl here anyway.- </AlertDescription>- </Alert>- );- }-- const markers = await Promise.all(+const getMarkers = (domain: string) =>+ Promise.all(Object.entries(REGIONS).map(async ([code, data]) => {const resolver = new InternalDoHResolver(code, 'cloudflare');// TODO Optimize this to only required records@@ -70,10 +31,13 @@ const MapResultsPage: FC<MapResultsPageProps> = async ({}));+const getHasDifferences = (+ markers: { A: string[]; AAAA: string[]; CNAME: string[] }[]+) => {let hasDifferentRecords = false;for (let i = 1; i < markers.length; i++) {- const previous = markers[i - 1].results;- const current = markers[i].results;+ const previous = markers[i - 1];+ const current = markers[i];if (previous.A.toSorted().toString() !== current.A.toSorted().toString() ||previous.AAAA.toSorted().toString() !==@@ -85,32 +49,62 @@ const MapResultsPage: FC<MapResultsPageProps> = async ({break;}}+ return hasDifferentRecords;+};++type MapResultsPageProps = {+ params: {+ domain: string;+ };+};++export const generateMetadata = ({+ params: { domain },+}: MapResultsPageProps): Metadata => ({+ openGraph: {+ url: `/lookup/${domain}/map`,+ },+ alternates: {+ canonical: `/lookup/${domain}/map`,+ },+});++const MapResultsPage: FC<MapResultsPageProps> = async ({+ params: { domain },+}) => {+ const { isBot, userAgent } = isUserBot(headers());++ if (isBot) {+ console.log('Bot detected, blocking request, UA:', userAgent);+ return (+ <BaseAlert icon={ShieldAlertIcon} title="Bot or crawler detected!">+ To protect our infrastructure, this page is not available for bots or+ crawlers.+ <br />+ But don't be sad, there's nothing to crawl here anyway.+ </BaseAlert>+ );+ }++ const markers = await getMarkers(domain);+ const hasDifferences = getHasDifferences(markers.map((m) => m.results));return (<>- <Alert className="mx-auto mt-16 max-w-max">- {hasDifferentRecords ? (- <>- <InfoIcon className="h-4 w-4" />- <AlertTitle>Different records detected!</AlertTitle>- <AlertDescription>- Not all regions have the same records for this domain. This{' '}- <i>could</i> be an indication for the use of GeoDNS.- <br /> Keep in mind however, that some providers rotate their IP- addresses, which can also lead to different results.- </AlertDescription>- </>- ) : (- <>- <CheckCircleIcon className="h-4 w-4" />- <AlertTitle>All records are the same!</AlertTitle>- <AlertDescription>- All records are the same for all regions. Therefore propagation- was successful and the domain is not using GeoDNS.- </AlertDescription>- </>- )}- </Alert>+ {hasDifferences ? (+ <BaseAlert icon={InfoIcon} title="Different records detected!">+ Not all regions have the same records for this domain. This{' '}+ <i>could</i> be an indication for the use of GeoDNS.+ <br /> Keep in mind however, that some providers rotate their IP+ addresses, which can also lead to different results.+ </BaseAlert>+ ) : (+ <BaseAlert icon={CheckCircleIcon} title="All records are the same!">+ All records are the same for all regions. Therefore propagation was+ successful and the domain is not using GeoDNS.+ </BaseAlert>+ )}+<ResultsGlobe domain={domain} markers={markers} /></>);diff --git a/app/lookup/[domain]/subdomains/page.tsx b/app/lookup/[domain]/subdomains/page.tsxindex 430bef3..a1e1bb3 100644--- a/app/lookup/[domain]/subdomains/page.tsx+++ b/app/lookup/[domain]/subdomains/page.tsx@@ -11,9 +11,8 @@ import {TableRow,} from '@/components/ui/table';-import { lookupCerts } from '@/lib/certs';import CloudflareDoHResolver from '@/lib/resolvers/CloudflareDoHResolver';-import { isValidDomain } from '@/lib/utils';+import { findSubdomains } from '@/lib/subdomains';import DomainLink from '../_components/DomainLink';@@ -41,42 +40,12 @@ export const generateMetadata = ({const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({params: { domain },}) => {- const resolver = new CloudflareDoHResolver();-- const certs = await lookupCerts(domain);-- const issuedCerts = certs.map((cert) => ({- date: new Date(cert.entry_timestamp),- domains: [cert.common_name, ...cert.name_value.split(/\n/g)],- }));-- const uniqueDomains = Array.from(- new Set<string>(issuedCerts.flatMap((r) => r.domains))- )- .filter(isValidDomain)- .filter((d) => d.endsWith(`.${domain}`));-- const results = await Promise.all(- // Limited to avoid subrequest limit from Cloudflare Workers of 1000- // https://developers.cloudflare.com/workers/platform/limits#subrequests- uniqueDomains.slice(0, 500).map(async (domain) => {- const records = await resolver.resolveRecordType(domain, 'A');- const hasRecords = records.length > 0;-- return {- domain,- firstSeen: issuedCerts- .filter((c) => c.domains.includes(domain))- .sort((a, b) => a.date.getTime() - b.date.getTime())[0].date,- stillExists: hasRecords,- };- })- );- const sortedResults = results.sort(- (a, b) => b.firstSeen.getTime() - a.firstSeen.getTime()+ const { results, isTruncated, RESULTS_LIMIT } = await findSubdomains(+ domain,+ new CloudflareDoHResolver());- if (!sortedResults.length) {+ if (!results.length) {return (<p className="mt-8 text-center text-muted-foreground">Could not find any subdomains for this domain.@@ -95,7 +64,7 @@ const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({</TableRow></TableHeader><TableBody>- {sortedResults.map((result) => (+ {results.map((result) => (<TableRow key={result.domain} className="hover:bg-transparent"><TableCell className="pl-0"><DomainLink domain={result.domain} />@@ -113,9 +82,9 @@ const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({</TableBody></Table>- {uniqueDomains.length > 500 && (+ {isTruncated && (<p className="mt-8 text-center text-muted-foreground">- Results limited to 500 subdomains.+ Results limited to {RESULTS_LIMIT} subdomains.</p>)}</>diff --git a/lib/api.ts b/lib/api.tsindex 85b1fa1..3aad467 100644--- a/lib/api.ts+++ b/lib/api.ts@@ -1,5 +1,7 @@import { Ratelimit } from '@upstash/ratelimit';import { Redis } from '@upstash/redis';+import { isbot } from 'isbot';+import type { ReadonlyHeaders } from 'next/dist/server/web/spec-extension/adapters/headers';type Unit = 'ms' | 's' | 'm' | 'h' | 'd';type Duration = `${number} ${Unit}` | `${number}${Unit}`;@@ -17,3 +19,9 @@ export const applyRateLimit = async (const result = await limiter.limit(identifier);return result.success;};++export const isUserBot = (headers: ReadonlyHeaders) => {+ const userAgent = headers.get('user-agent');+ const isBot = !userAgent || isbot(userAgent);+ return { isBot, userAgent };+};diff --git a/lib/subdomains.ts b/lib/subdomains.tsnew file mode 100644index 0000000..26b5ee4--- /dev/null+++ b/lib/subdomains.ts@@ -0,0 +1,47 @@+import { lookupCerts } from './certs';+import type DnsResolver from './resolvers/DnsResolver';+import { isValidDomain } from './utils';++const RESULTS_LIMIT = 500;++export const findSubdomains = async (domain: string, resolver: DnsResolver) => {+ const certs = await lookupCerts(domain);++ const issuedCerts = certs.map((cert) => ({+ date: new Date(cert.entry_timestamp),+ domains: [cert.common_name, ...cert.name_value.split(/\n/g)],+ }));++ const uniqueDomains = Array.from(+ new Set<string>(issuedCerts.flatMap((r) => r.domains))+ )+ .filter(isValidDomain)+ .filter((d) => d.endsWith(`.${domain}`));++ const results = await Promise.all(+ // Limited to avoid subrequest limit from Cloudflare Workers of 1000+ // https://developers.cloudflare.com/workers/platform/limits#subrequests+ uniqueDomains.slice(0, RESULTS_LIMIT).map(async (domain) => {+ const records = await resolver.resolveRecordType(domain, 'A');+ const hasRecords = records.length > 0;++ return {+ domain,+ firstSeen: issuedCerts+ .filter((c) => c.domains.includes(domain))+ .sort((a, b) => a.date.getTime() - b.date.getTime())[0].date,+ stillExists: hasRecords,+ };+ })+ );++ const sortedResults = results.toSorted(+ (a, b) => b.firstSeen.getTime() - a.firstSeen.getTime()+ );++ return {+ results: sortedResults,+ isTruncated: uniqueDomains.length > RESULTS_LIMIT,+ RESULTS_LIMIT,+ };+};