Full Toolkit for Next-Level Domain Analysis
Remove Upstash-based rate limiting 🚮
Felix Wotschofskyd4d72efb66f3
diff --git a/.env.example b/.env.exampleindex e1b2cd4..ede6653 100644--- a/.env.example+++ b/.env.example@@ -8,10 +8,6 @@ INTERNAL_API_SECRET=secretNEXT_PUBLIC_PLAUSIBLE_DOMAIN=example.comNEXT_PUBLIC_PLAUSIBLE_HOST=https://customplausible.io # Optional if using plausible.io-# Credentials for Upstash Redis database-UPSTASH_REDIS_REST_URL=https://xyz.upstash.io-UPSTASH_REDIS_REST_TOKEN=secret-# GCP JSON Service Account encoded as base64GOOGLE_SERVICE_KEY_B64=ew...# Name of the BigQuery datasetdiff --git a/SETUP.md b/SETUP.mdindex 53fa588..927f39a 100644--- a/SETUP.md+++ b/SETUP.md@@ -14,7 +14,7 @@ The codebase can and should be formatted with Prettier using `pnpm format`.## Configuring the project-For full functionality, Domain Digger relies on Upstash Redis and Google BigQuery.+For full functionality, Domain Digger relies on Google BigQuery.Credentials are provided through environment variables. Using [a .env file as per the Next.js docs](https://nextjs.org/docs/basic-features/environment-variables#loading-environment-variables) is recommended. All required variables are documented in the [example env file](./.env.example).@@ -24,10 +24,6 @@ As of writing this, Vercel is the only platform able to correctly deploy DomainWhile a traditional non-serverless deployment is possible, the DNS map won't work correctly.-### Upstash--Head to the [Upstash Console](https://console.upstash.com) and create a Redis database from there. Retrieve credentials for `UPSTASH_REDIS_REST_URL` and `UPSTASH_REDIS_REST_TOKEN` (scroll down to the "REST API" section).-### Google CloudConfiguring Google Cloud _is not_ required to run the codebase.diff --git a/app/(errors)/too-many-requests/page.tsx b/app/(errors)/too-many-requests/page.tsxdeleted file mode 100644index 998e2a8..0000000--- a/app/(errors)/too-many-requests/page.tsx+++ /dev/null@@ -1,12 +0,0 @@-import { type FC } from 'react';--const TooManyRequestsPage: FC = () => (- <div className="mt-12 flex flex-col items-center gap-2">- <h2>429 - Too Many Requests</h2>- <p className="mt-2 text-center text-sm text-muted-foreground">- Please try again later- </p>- </div>-);--export default TooManyRequestsPage;diff --git a/app/api/ip-details/route.ts b/app/api/ip-details/route.tsindex caab990..daad6b3 100644--- a/app/api/ip-details/route.ts+++ b/app/api/ip-details/route.ts@@ -1,7 +1,6 @@import { NextResponse } from 'next/server';import isIP from 'validator/lib/isIP';-import { applyRateLimit } from '@/lib/api';import { getIpDetails, lookupReverse } from '@/lib/ips';export const runtime = 'edge';@@ -32,15 +31,6 @@ export async function GET(request: Request) {);}- const visitorIp = request.headers.get('x-forwarded-for') ?? '';- const identifier = ['ip-details', visitorIp].join(':');- const isAllowed = await applyRateLimit(identifier, 10, '60s');- if (!isAllowed) {- return new Response('Too many requests', {- status: 429,- });- }-const [data, reverse] = await Promise.all([getIpDetails(ip),lookupReverse(ip),diff --git a/app/api/whois-summary/route.ts b/app/api/whois-summary/route.tsindex ab95a05..fc42616 100644--- a/app/api/whois-summary/route.ts+++ b/app/api/whois-summary/route.ts@@ -1,4 +1,3 @@-import { applyRateLimit } from '@/lib/api';import { getWhoisSummary } from '@/lib/whois';export type WhoisSummaryResponse = Awaited<ReturnType<typeof getWhoisSummary>>;@@ -20,15 +19,6 @@ export async function GET(request: Request) {);}- const visitorIp = request.headers.get('x-forwarded-for') ?? '';- const identifier = ['whois-summary', visitorIp].join(':');- const isAllowed = await applyRateLimit(identifier, 5, '60s');- if (!isAllowed) {- return new Response('Too many requests', {- status: 429,- });- }-try {const summary = await getWhoisSummary(domain);return Response.json(summary, {diff --git a/env.ts b/env.tsindex 8a0320b..e351ca9 100644--- a/env.ts+++ b/env.ts@@ -18,8 +18,6 @@ export const env = createEnv({SITE_URL: z.string().url().optional(),INTERNAL_API_SECRET: z.string().min(1),- UPSTASH_REDIS_REST_URL: z.string().url(),- UPSTASH_REDIS_REST_TOKEN: z.string().min(1),ALLOWED_BOTS: z.string().transform(safeJSONParse)diff --git a/lib/api.ts b/lib/api.tsindex 47d546c..84cdc31 100644--- a/lib/api.ts+++ b/lib/api.ts@@ -1,32 +1,12 @@-import { Ratelimit } from '@upstash/ratelimit';-import { Redis } from '@upstash/redis';import { isbot } from 'isbot';-import type { ReadonlyHeaders } from 'next/dist/server/web/spec-extension/adapters/headers';-type Unit = 'ms' | 's' | 'm' | 'h' | 'd';-type Duration = `${number} ${Unit}` | `${number}${Unit}`;--export const applyRateLimit = async (- identifier: string,- tokens: number,- window: Duration,-) => {- const limiter = new Ratelimit({- redis: Redis.fromEnv(),- limiter: Ratelimit.slidingWindow(tokens, window),- analytics: true,- });- const result = await limiter.limit(identifier);- return result.success;-};--export const getVisitorIp = (headers: ReadonlyHeaders) => {+export const getVisitorIp = (headers: Headers) => {const forwardedFor = headers.get('x-forwarded-for');const ip = (forwardedFor ?? '127.0.0.1').split(',')[0];return ip;};-export const isUserBot = (headers: ReadonlyHeaders) => {+export const isUserBot = (headers: Headers) => {const userAgent = headers.get('user-agent');const isBot = !userAgent || isbot(userAgent);return { isBot, userAgent };diff --git a/middleware.ts b/middleware.tsindex a9733a2..93ac758 100644--- a/middleware.ts+++ b/middleware.ts@@ -1,7 +1,7 @@import { type NextRequest, NextResponse } from 'next/server';import { env } from '@/env';-import { applyRateLimit, isUserBot } from '@/lib/api';+import { isUserBot } from '@/lib/api';export const middleware = async (request: NextRequest) => {if (env.ALLOWED_BOTS) {@@ -17,16 +17,6 @@ export const middleware = async (request: NextRequest) => {}}- const visitorIp = request.headers.get('x-forwarded-for') ?? '';- const identifier = ['lookup-page', visitorIp].join(':');- const isAllowed = await applyRateLimit(identifier, 10, '60s');-- if (!isAllowed) {- return NextResponse.rewrite(new URL('/too-many-requests', request.url), {- status: 429,- });- }-return NextResponse.next();};diff --git a/package.json b/package.jsonindex 2c21f82..a0d6258 100644--- a/package.json+++ b/package.json@@ -28,8 +28,6 @@"@types/punycode": "^2.1.4","@types/validator": "^13.12.0","@uidotdev/usehooks": "^2.4.1",- "@upstash/ratelimit": "^2.0.1",- "@upstash/redis": "^1.34.0","@vercel/speed-insights": "^1.0.12","autoprefixer": "^10.4.20","class-variance-authority": "^0.7.0",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex dd4f1a4..5bdd801 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -47,12 +47,6 @@ importers:'@uidotdev/usehooks':specifier: ^2.4.1- '@upstash/ratelimit':- specifier: ^2.0.1- version: 2.0.1- '@upstash/redis':- specifier: ^1.34.0- version: 1.34.0'@vercel/speed-insights':specifier: ^1.0.12version: 1.0.12([email protected](@babel/[email protected])([email protected]([email protected]))([email protected]))([email protected])@@ -1902,16 +1896,6 @@ packages:'@ungap/[email protected]':resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}- '@upstash/[email protected]':- resolution: {integrity: sha512-7qJHGxpQgQr9/vmeS1PktEwvNAF7TI4iJDi8Pu2CFZ9YUGHZH4fOP5TfYlZ4aVxfopnELiE4BS4FBjyK7V1/xQ==}- engines: {node: '>=16.0.0'}-- '@upstash/[email protected]':- resolution: {integrity: sha512-J+0hlkvWUjlVrjcBQhWx7gbaUGsvFF59i+GAx7YQk8L0E0MQ93xzCPu02uaXhGDJGkxiar7nRRPqj3hs+CdAJg==}-- '@upstash/[email protected]':- resolution: {integrity: sha512-TrXNoJLkysIl8SBc4u9bNnyoFYoILpCcFJcLyWCccb/QSUmaVKdvY0m5diZqc3btExsapcMbaw/s/wh9Sf1pJw==}-'@vercel/[email protected]':resolution: {integrity: sha512-ZGQ+a7bcfWJD2VYEp2R1LHvRAMyyaFBYytZXsfnbOMkeOvzGNVxUL7aVUvisIrTZjXTSsxG45DKX7yiw6nq2Jw==}peerDependencies:@@ -2221,9 +2205,6 @@ packages:resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}engines: {node: '>= 8'}- resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==}-resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==}@@ -5947,18 +5928,6 @@ snapshots:'@ungap/[email protected]': {}- '@upstash/[email protected]':- dependencies:- '@upstash/redis': 1.34.0-- '@upstash/[email protected]':- dependencies:- '@upstash/core-analytics': 0.0.10-- '@upstash/[email protected]':- dependencies:- crypto-js: 4.2.0-'@vercel/[email protected]([email protected](@babel/[email protected])([email protected]([email protected]))([email protected]))([email protected])':optionalDependencies:@@ -6296,8 +6265,6 @@ snapshots:shebang-command: 2.0.0which: 2.0.2- [email protected]: {}-dependencies:boolbase: 1.0.0