Skip to content

Full Toolkit for Next-Level Domain Analysis

More refactoring 🏗️

Felix Wotschofskyb4841b8ed802
diff --git a/app/lookup/[domain]/map/_components/BaseAlert.tsx b/app/lookup/[domain]/map/_components/BaseAlert.tsx
new file mode 100644
index 0000000..064a001
--- /dev/null
+++ b/app/lookup/[domain]/map/_components/BaseAlert.tsx
@@ -0,0 +1,20 @@
+import type { LucideIcon } from 'lucide-react';
+import type { FC } from 'react';
+
+import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
+
+type BaseAlertProps = {
+ icon: LucideIcon;
+ title: string;
+ children: React.ReactNode;
+};
+
+const BaseAlert: FC<BaseAlertProps> = ({ icon: Icon, title, children }) => (
+ <Alert className="mx-auto mt-24 max-w-max">
+ <Icon className="h-4 w-4" />
+ <AlertTitle>{title}</AlertTitle>
+ <AlertDescription>{children}</AlertDescription>
+ </Alert>
+);
+
+export default BaseAlert;
diff --git a/app/lookup/[domain]/map/page.tsx b/app/lookup/[domain]/map/page.tsx
index 520306b..c12c95a 100644
--- a/app/lookup/[domain]/map/page.tsx
+++ b/app/lookup/[domain]/map/page.tsx
@@ -1,58 +1,19 @@
-import { isbot } from 'isbot';
import { CheckCircleIcon, InfoIcon, ShieldAlertIcon } from 'lucide-react';
import type { Metadata } from 'next';
import { headers } from 'next/headers';
import type { FC } from 'react';
-import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
-
+import { isUserBot } from '@/lib/api';
import { REGIONS } from '@/lib/data';
import InternalDoHResolver from '@/lib/resolvers/InternalDoHResolver';
+import BaseAlert from './_components/BaseAlert';
import ResultsGlobe from './_components/ResultsGlobe';
export const runtime = 'edge';
-type MapResultsPageProps = {
- params: {
- domain: string;
- };
-};
-
-export const generateMetadata = ({
- params: { domain },
-}: MapResultsPageProps): Metadata => ({
- openGraph: {
- url: `/lookup/${domain}/map`,
- },
- alternates: {
- canonical: `/lookup/${domain}/map`,
- },
-});
-
-const MapResultsPage: FC<MapResultsPageProps> = async ({
- params: { domain },
-}) => {
- const userAgent = headers().get('user-agent');
- const shouldBlockRequest = !userAgent || isbot(userAgent);
-
- if (shouldBlockRequest) {
- console.log('Bot detected, blocking request, UA:', userAgent);
- return (
- <Alert className="mx-auto mt-24 max-w-max">
- <ShieldAlertIcon className="h-4 w-4" />
- <AlertTitle>Bot or crawler detected!</AlertTitle>
- <AlertDescription>
- To protect our infrastructure, this page is not available for bots or
- crawlers.
- <br />
- But don&apos;t be sad, there&apos;s to crawl here anyway.
- </AlertDescription>
- </Alert>
- );
- }
-
- const markers = await Promise.all(
+const getMarkers = (domain: string) =>
+ Promise.all(
Object.entries(REGIONS).map(async ([code, data]) => {
const resolver = new InternalDoHResolver(code, 'cloudflare');
// TODO Optimize this to only required records
@@ -70,10 +31,13 @@ const MapResultsPage: FC<MapResultsPageProps> = async ({
})
);
+const getHasDifferences = (
+ markers: { A: string[]; AAAA: string[]; CNAME: string[] }[]
+) => {
let hasDifferentRecords = false;
for (let i = 1; i < markers.length; i++) {
- const previous = markers[i - 1].results;
- const current = markers[i].results;
+ const previous = markers[i - 1];
+ const current = markers[i];
if (
previous.A.toSorted().toString() !== current.A.toSorted().toString() ||
previous.AAAA.toSorted().toString() !==
@@ -85,32 +49,62 @@ const MapResultsPage: FC<MapResultsPageProps> = async ({
break;
}
}
+ return hasDifferentRecords;
+};
+
+type MapResultsPageProps = {
+ params: {
+ domain: string;
+ };
+};
+
+export const generateMetadata = ({
+ params: { domain },
+}: MapResultsPageProps): Metadata => ({
+ openGraph: {
+ url: `/lookup/${domain}/map`,
+ },
+ alternates: {
+ canonical: `/lookup/${domain}/map`,
+ },
+});
+
+const MapResultsPage: FC<MapResultsPageProps> = async ({
+ params: { domain },
+}) => {
+ const { isBot, userAgent } = isUserBot(headers());
+
+ if (isBot) {
+ console.log('Bot detected, blocking request, UA:', userAgent);
+ return (
+ <BaseAlert icon={ShieldAlertIcon} title="Bot or crawler detected!">
+ To protect our infrastructure, this page is not available for bots or
+ crawlers.
+ <br />
+ But don&apos;t be sad, there&apos;s nothing to crawl here anyway.
+ </BaseAlert>
+ );
+ }
+
+ const markers = await getMarkers(domain);
+ const hasDifferences = getHasDifferences(markers.map((m) => m.results));
return (
<>
- <Alert className="mx-auto mt-16 max-w-max">
- {hasDifferentRecords ? (
- <>
- <InfoIcon className="h-4 w-4" />
- <AlertTitle>Different records detected!</AlertTitle>
- <AlertDescription>
- Not all regions have the same records for this domain. This{' '}
- <i>could</i> be an indication for the use of GeoDNS.
- <br /> Keep in mind however, that some providers rotate their IP
- addresses, which can also lead to different results.
- </AlertDescription>
- </>
- ) : (
- <>
- <CheckCircleIcon className="h-4 w-4" />
- <AlertTitle>All records are the same!</AlertTitle>
- <AlertDescription>
- All records are the same for all regions. Therefore propagation
- was successful and the domain is not using GeoDNS.
- </AlertDescription>
- </>
- )}
- </Alert>
+ {hasDifferences ? (
+ <BaseAlert icon={InfoIcon} title="Different records detected!">
+ Not all regions have the same records for this domain. This{' '}
+ <i>could</i> be an indication for the use of GeoDNS.
+ <br /> Keep in mind however, that some providers rotate their IP
+ addresses, which can also lead to different results.
+ </BaseAlert>
+ ) : (
+ <BaseAlert icon={CheckCircleIcon} title="All records are the same!">
+ All records are the same for all regions. Therefore propagation was
+ successful and the domain is not using GeoDNS.
+ </BaseAlert>
+ )}
+
<ResultsGlobe domain={domain} markers={markers} />
</>
);
diff --git a/app/lookup/[domain]/subdomains/page.tsx b/app/lookup/[domain]/subdomains/page.tsx
index 430bef3..a1e1bb3 100644
--- a/app/lookup/[domain]/subdomains/page.tsx
+++ b/app/lookup/[domain]/subdomains/page.tsx
@@ -11,9 +11,8 @@ import {
TableRow,
} from '@/components/ui/table';
-import { lookupCerts } from '@/lib/certs';
import CloudflareDoHResolver from '@/lib/resolvers/CloudflareDoHResolver';
-import { isValidDomain } from '@/lib/utils';
+import { findSubdomains } from '@/lib/subdomains';
import DomainLink from '../_components/DomainLink';
@@ -41,42 +40,12 @@ export const generateMetadata = ({
const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({
params: { domain },
}) => {
- const resolver = new CloudflareDoHResolver();
-
- const certs = await lookupCerts(domain);
-
- const issuedCerts = certs.map((cert) => ({
- date: new Date(cert.entry_timestamp),
- domains: [cert.common_name, ...cert.name_value.split(/\n/g)],
- }));
-
- const uniqueDomains = Array.from(
- new Set<string>(issuedCerts.flatMap((r) => r.domains))
- )
- .filter(isValidDomain)
- .filter((d) => d.endsWith(`.${domain}`));
-
- const results = await Promise.all(
- // Limited to avoid subrequest limit from Cloudflare Workers of 1000
- // https://developers.cloudflare.com/workers/platform/limits#subrequests
- uniqueDomains.slice(0, 500).map(async (domain) => {
- const records = await resolver.resolveRecordType(domain, 'A');
- const hasRecords = records.length > 0;
-
- return {
- domain,
- firstSeen: issuedCerts
- .filter((c) => c.domains.includes(domain))
- .sort((a, b) => a.date.getTime() - b.date.getTime())[0].date,
- stillExists: hasRecords,
- };
- })
- );
- const sortedResults = results.sort(
- (a, b) => b.firstSeen.getTime() - a.firstSeen.getTime()
+ const { results, isTruncated, RESULTS_LIMIT } = await findSubdomains(
+ domain,
+ new CloudflareDoHResolver()
);
- if (!sortedResults.length) {
+ if (!results.length) {
return (
<p className="mt-8 text-center text-muted-foreground">
Could not find any subdomains for this domain.
@@ -95,7 +64,7 @@ const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({
</TableRow>
</TableHeader>
<TableBody>
- {sortedResults.map((result) => (
+ {results.map((result) => (
<TableRow key={result.domain} className="hover:bg-transparent">
<TableCell className="pl-0">
<DomainLink domain={result.domain} />
@@ -113,9 +82,9 @@ const SubdomainsResultsPage: FC<SubdomainsResultsPageProps> = async ({
</TableBody>
</Table>
- {uniqueDomains.length > 500 && (
+ {isTruncated && (
<p className="mt-8 text-center text-muted-foreground">
- Results limited to 500 subdomains.
+ Results limited to {RESULTS_LIMIT} subdomains.
</p>
)}
</>
diff --git a/lib/api.ts b/lib/api.ts
index 85b1fa1..3aad467 100644
--- a/lib/api.ts
+++ b/lib/api.ts
@@ -1,5 +1,7 @@
import { Ratelimit } from '@upstash/ratelimit';
import { Redis } from '@upstash/redis';
+import { isbot } from 'isbot';
+import type { ReadonlyHeaders } from 'next/dist/server/web/spec-extension/adapters/headers';
type Unit = 'ms' | 's' | 'm' | 'h' | 'd';
type Duration = `${number} ${Unit}` | `${number}${Unit}`;
@@ -17,3 +19,9 @@ export const applyRateLimit = async (
const result = await limiter.limit(identifier);
return result.success;
};
+
+export const isUserBot = (headers: ReadonlyHeaders) => {
+ const userAgent = headers.get('user-agent');
+ const isBot = !userAgent || isbot(userAgent);
+ return { isBot, userAgent };
+};
diff --git a/lib/subdomains.ts b/lib/subdomains.ts
new file mode 100644
index 0000000..26b5ee4
--- /dev/null
+++ b/lib/subdomains.ts
@@ -0,0 +1,47 @@
+import { lookupCerts } from './certs';
+import type DnsResolver from './resolvers/DnsResolver';
+import { isValidDomain } from './utils';
+
+const RESULTS_LIMIT = 500;
+
+export const findSubdomains = async (domain: string, resolver: DnsResolver) => {
+ const certs = await lookupCerts(domain);
+
+ const issuedCerts = certs.map((cert) => ({
+ date: new Date(cert.entry_timestamp),
+ domains: [cert.common_name, ...cert.name_value.split(/\n/g)],
+ }));
+
+ const uniqueDomains = Array.from(
+ new Set<string>(issuedCerts.flatMap((r) => r.domains))
+ )
+ .filter(isValidDomain)
+ .filter((d) => d.endsWith(`.${domain}`));
+
+ const results = await Promise.all(
+ // Limited to avoid subrequest limit from Cloudflare Workers of 1000
+ // https://developers.cloudflare.com/workers/platform/limits#subrequests
+ uniqueDomains.slice(0, RESULTS_LIMIT).map(async (domain) => {
+ const records = await resolver.resolveRecordType(domain, 'A');
+ const hasRecords = records.length > 0;
+
+ return {
+ domain,
+ firstSeen: issuedCerts
+ .filter((c) => c.domains.includes(domain))
+ .sort((a, b) => a.date.getTime() - b.date.getTime())[0].date,
+ stillExists: hasRecords,
+ };
+ })
+ );
+
+ const sortedResults = results.toSorted(
+ (a, b) => b.firstSeen.getTime() - a.firstSeen.getTime()
+ );
+
+ return {
+ results: sortedResults,
+ isTruncated: uniqueDomains.length > RESULTS_LIMIT,
+ RESULTS_LIMIT,
+ };
+};