Skip to content

Full Toolkit for Next-Level Domain Analysis

Log requests to BigQuery 📜

Felix Wotschofskyadd7e3bb5cf2
diff --git a/app/lookup/[domain]/layout.tsx b/app/lookup/[domain]/layout.tsx
index a1101a4..1b16161 100644
--- a/app/lookup/[domain]/layout.tsx
+++ b/app/lookup/[domain]/layout.tsx
@@ -1,14 +1,17 @@
import { ExternalLinkIcon } from 'lucide-react';
import type { Metadata } from 'next';
import dynamic from 'next/dynamic';
+import { headers } from 'next/headers';
import { notFound } from 'next/navigation';
import { type FC, type ReactNode } from 'react';
+import { getDomain } from 'tldts';
import SearchForm from '@/components/general/SearchForm';
import RelatedDomains from '@/components/results/RelatedDomains';
import ResultsTabs from '@/components/results/ResultsTabs';
import ShareButton from '@/components/results/ShareButton';
import WhoisQuickInfo from '@/components/results/WhoisQuickInfo';
+import bigquery from '@/lib/bigquery';
import { isValidDomain } from '@/lib/utils';
const StarReminder = dynamic(
@@ -43,6 +46,36 @@ const LookupLayout: FC<LookupLayoutProps> = ({
return notFound();
}
+ if (bigquery) {
+ const baseDomain = getDomain(domain);
+
+ const forwardedFor = headers().get('x-forwarded-for');
+ const ip = (forwardedFor ?? '127.0.0.1').split(',')[0];
+
+ bigquery
+ .insertRows({
+ datasetName: process.env.BIGQUERY_DATASET!,
+ tableName: 'lookups',
+ rows: [
+ {
+ domain,
+ baseDomain,
+ timestamp: Math.floor(new Date().getTime() / 1000),
+ ip,
+ },
+ ],
+ })
+ .catch((error) => {
+ if ('errors' in error) {
+ for (const err of error.errors) {
+ console.error(err);
+ }
+ } else {
+ console.error(error);
+ }
+ });
+ }
+
return (
<>
<div className="container mb-8 max-w-xl">
diff --git a/lib/bigquery.ts b/lib/bigquery.ts
new file mode 100644
index 0000000..9fd9a44
--- /dev/null
+++ b/lib/bigquery.ts
@@ -0,0 +1,55 @@
+// @ts-expect-error
+import { getAccessToken } from 'web-auth-library/google';
+
+const credentials = process.env.GOOGLE_SERVICE_KEY_B64
+ ? (JSON.parse(
+ Buffer.from(process.env.GOOGLE_SERVICE_KEY_B64, 'base64').toString()
+ ) as {
+ type: 'service_account';
+ project_id: string;
+ private_key: string;
+ client_email: string;
+ })
+ : null;
+
+export const insertRows = async ({
+ datasetName,
+ tableName,
+ rows,
+}: {
+ datasetName: string;
+ tableName: string;
+ rows: Record<string, any>[];
+}): Promise<void> => {
+ const accessToken = await getAccessToken({
+ credentials: credentials,
+ scope: 'https://www.googleapis.com/auth/cloud-platform',
+ });
+
+ const response = await fetch(
+ `https://bigquery.googleapis.com/bigquery/v2/projects/${
+ credentials!.project_id
+ }/datasets/${datasetName}/tables/${tableName}/insertAll`,
+ {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ rows: rows.map((row) => ({ json: row })),
+ }),
+ }
+ );
+
+ if (!response.ok) {
+ const errorBody = await response.text();
+ throw new Error(
+ `Failed to insert data into BigQuery: ${response.status} ${errorBody}`
+ );
+ }
+};
+
+export default credentials && {
+ insertRows,
+};
diff --git a/package.json b/package.json
index b4f2fd1..1db1346 100644
--- a/package.json
+++ b/package.json
@@ -58,6 +58,7 @@
"tldts": "^6.1.4",
"validator": "^13.11.0",
"vaul": "^0.8.0",
+ "web-auth-library": "^1.0.3",
"whoiser": "^1.17.1"
},
"devDependencies": {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 5b465e9..6515610 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -137,6 +137,9 @@ dependencies:
vaul:
specifier: ^0.8.0
+ web-auth-library:
+ specifier: ^1.0.3
+ version: 1.0.3
whoiser:
specifier: ^1.17.1
version: 1.17.1
@@ -4438,6 +4441,10 @@ packages:
hasBin: true
dev: false
+ resolution: {integrity: sha512-W+oqK4H+r5sITxfxpSU+MMdr/YSWGvgZMQDIsNoBDGGy4i7GBPTtvFKibQzW06n3U3TqHjhvBJsirShsEJ6eeQ==}
+ dev: false
+
resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==}
@@ -5319,6 +5326,10 @@ packages:
resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
engines: {iojs: '>=1.0.0', node: '>=0.10.0'}
+ resolution: {integrity: sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ==}
+ dev: false
+
resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==}
hasBin: true
@@ -5959,6 +5970,13 @@ packages:
graceful-fs: 4.2.11
dev: false
+ resolution: {integrity: sha512-fBmEjJSrbmbD9EREwVlewyfSVkb3IzgTXEF0fzXo3miDywsxES1vwG4aJGNpuSSUorZAGBJNyyz5VFq2VFgudw==}
+ dependencies:
+ jose: 4.15.4
+ rfc4648: 1.5.3
+ dev: false
+
resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==}
dependencies: