Skip to content

Full Toolkit for Next-Level Domain Analysis

Record bot detection result in BigQuery 🤖

Felix Wotschofsky2d2921b46431
diff --git a/SETUP.md b/SETUP.md
index b5a0fcd..ff66cd6 100644
--- a/SETUP.md
+++ b/SETUP.md
@@ -63,6 +63,13 @@ Use the following schema for creating the dataset:
"type": "STRING",
"description": null,
"fields": []
+ },
+ {
+ "name": "isBot",
+ "mode": "NULLABLE",
+ "type": "BOOLEAN",
+ "description": null,
+ "fields": []
}
]
```
diff --git a/app/lookup/[domain]/layout.tsx b/app/lookup/[domain]/layout.tsx
index 08412ea..db82639 100644
--- a/app/lookup/[domain]/layout.tsx
+++ b/app/lookup/[domain]/layout.tsx
@@ -1,9 +1,11 @@
import { ExternalLinkIcon } from 'lucide-react';
import type { Metadata } from 'next';
import dynamic from 'next/dynamic';
+import { headers } from 'next/headers';
import { notFound } from 'next/navigation';
import { type FC, type ReactNode } from 'react';
+import { isUserBot } from '@/lib/api';
import { recordLookup } from '@/lib/search';
import { isValidDomain } from '@/lib/utils';
@@ -44,7 +46,8 @@ const LookupLayout: FC<LookupLayoutProps> = ({
return notFound();
}
- recordLookup(domain);
+ const { isBot } = isUserBot(headers());
+ recordLookup({ domain, isBot });
return (
<>
diff --git a/lib/search.ts b/lib/search.ts
index b109753..9a316a0 100644
--- a/lib/search.ts
+++ b/lib/search.ts
@@ -3,12 +3,12 @@ import { getDomain } from 'tldts';
import { bigquery } from '@/lib/bigquery';
-export const recordLookup = (domain: string) => {
+export const recordLookup = (data: { domain: string; isBot: boolean }) => {
if (!bigquery) {
return;
}
- const baseDomain = getDomain(domain);
+ const baseDomain = getDomain(data.domain);
const forwardedFor = headers().get('x-forwarded-for');
const ip = (forwardedFor ?? '127.0.0.1').split(',')[0];
@@ -19,10 +19,11 @@ export const recordLookup = (domain: string) => {
tableName: 'lookups',
rows: [
{
- domain,
+ domain: data.domain,
baseDomain,
timestamp: Math.floor(new Date().getTime() / 1000),
ip,
+ isBot: data.isBot,
},
],
})