Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

feat: add bot blocking feature to site settings and API (#274)

Bill Yang2981b4aea67a
diff --git a/client/src/api/admin/sites.ts b/client/src/api/admin/sites.ts
index e5850e1ec..b53f4eae3 100644
--- a/client/src/api/admin/sites.ts
+++ b/client/src/api/admin/sites.ts
@@ -12,6 +12,7 @@ export type SiteResponse = {
createdBy: string;
public: boolean;
saltUserIds: boolean;
+ blockBots: boolean;
isOwner: boolean;
};
@@ -24,6 +25,7 @@ export type GetSitesResponse = {
createdBy: string;
public: boolean;
saltUserIds: boolean;
+ blockBots: boolean;
overMonthlyLimit?: boolean;
monthlyEventCount?: number;
eventLimit?: number;
@@ -51,6 +53,7 @@ export function addSite(
settings?: {
isPublic?: boolean;
saltUserIds?: boolean;
+ blockBots?: boolean;
}
) {
return authedFetchWithError<{ siteId: number }>(`${BACKEND_URL}/add-site`, {
@@ -61,6 +64,7 @@ export function addSite(
organizationId,
public: settings?.isPublic || false,
saltUserIds: settings?.saltUserIds || false,
+ blockBots: settings?.blockBots === undefined ? true : settings?.blockBots,
}),
headers: {
"Content-Type": "application/json",
@@ -151,3 +155,16 @@ export function changeSiteSalt(siteId: number, saltUserIds: boolean) {
},
});
}
+
+export function changeSiteBlockBots(siteId: number, blockBots: boolean) {
+ return authedFetchWithError(`${BACKEND_URL}/change-site-block-bots`, {
+ method: "POST",
+ body: JSON.stringify({
+ siteId,
+ blockBots,
+ }),
+ headers: {
+ "Content-Type": "application/json",
+ },
+ });
+}
diff --git a/client/src/components/SiteSettings/SiteSettings.tsx b/client/src/components/SiteSettings/SiteSettings.tsx
index 371c11bb8..7f8a48299 100644
--- a/client/src/components/SiteSettings/SiteSettings.tsx
+++ b/client/src/components/SiteSettings/SiteSettings.tsx
@@ -36,6 +36,7 @@ import {
changeSiteDomain,
changeSitePublic,
changeSiteSalt,
+ changeSiteBlockBots,
deleteSite,
GetSitesResponse,
SiteResponse,
@@ -81,6 +82,10 @@ export function SiteSettingsInner({
const [isChangingPublic, setIsChangingPublic] = useState(false);
const [isSalting, setIsSalting] = useState(siteMetadata.saltUserIds || false);
const [isChangingSalt, setIsChangingSalt] = useState(false);
+ const [isBlockingBots, setIsBlockingBots] = useState(
+ siteMetadata.blockBots === undefined ? true : siteMetadata.blockBots
+ );
+ const [isChangingBlockBots, setIsChangingBlockBots] = useState(false);
const [isDeleting, setIsDeleting] = useState(false);
const [dialogOpen, setDialogOpen] = useState(false);
const [activeTab, setActiveTab] = useState("script");
@@ -158,6 +163,22 @@ export function SiteSettingsInner({
}
};
+ const handleBlockBotsToggle = async (checked: boolean) => {
+ try {
+ setIsChangingBlockBots(true);
+ await changeSiteBlockBots(siteMetadata.siteId, checked);
+ setIsBlockingBots(checked);
+ toast.success(checked ? "Bot blocking enabled" : "Bot blocking disabled");
+ refetch();
+ } catch (error) {
+ console.error("Error changing bot blocking setting:", error);
+ toast.error("Failed to update bot blocking setting");
+ setIsBlockingBots(!checked); // Revert UI state on error
+ } finally {
+ setIsChangingBlockBots(false);
+ }
+ };
+
if (!siteMetadata) {
return null;
}
@@ -239,6 +260,28 @@ export function SiteSettingsInner({
/>
</div>
+ {/* Bot Blocking Section */}
+ <div className="flex items-center justify-between">
+ <div>
+ <Label
+ htmlFor="blockBots"
+ className="text-sm font-medium text-foreground block"
+ >
+ Block Bot Traffic
+ </Label>
+ <p className="text-xs text-muted-foreground mt-1">
+ When enabled, traffic from known bots and crawlers will not be
+ tracked
+ </p>
+ </div>
+ <Switch
+ id="blockBots"
+ checked={isBlockingBots}
+ disabled={isChangingBlockBots || disabled}
+ onCheckedChange={handleBlockBotsToggle}
+ />
+ </div>
+
{/* Domain Settings Section */}
<div className="space-y-3">
<div>
diff --git a/server/package-lock.json b/server/package-lock.json
index 62e0080f6..080a20659 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -19,6 +19,7 @@
"drizzle-orm": "0.43.1",
"fastify": "5.3.3",
"fastify-better-auth": "1.0.4",
+ "isbot": "^5.1.28",
"luxon": "3.6.1",
"node-cron": "4.0.5",
"pg": "8.16.0",
@@ -2689,6 +2690,14 @@
],
"license": "MIT"
},
+ "node_modules/isbot": {
+ "version": "5.1.28",
+ "resolved": "https://registry.npmjs.org/isbot/-/isbot-5.1.28.tgz",
+ "integrity": "sha512-qrOp4g3xj8YNse4biorv6O5ZShwsJM0trsoda4y7j/Su7ZtTTfVXFzbKkpgcSoDrHS8FcTuUwcU04YimZlZOxw==",
+ "engines": {
+ "node": ">=18"
+ }
+ },
"node_modules/isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
diff --git a/server/package.json b/server/package.json
index a98732a13..2f852a8e4 100644
--- a/server/package.json
+++ b/server/package.json
@@ -27,6 +27,7 @@
"drizzle-orm": "0.43.1",
"fastify": "5.3.3",
"fastify-better-auth": "1.0.4",
+ "isbot": "^5.1.28",
"luxon": "3.6.1",
"node-cron": "4.0.5",
"pg": "8.16.0",
diff --git a/server/src/api/sites/addSite.ts b/server/src/api/sites/addSite.ts
index 6ad4b2e99..19c01afbd 100644
--- a/server/src/api/sites/addSite.ts
+++ b/server/src/api/sites/addSite.ts
@@ -13,6 +13,7 @@ export async function addSite(
organizationId: string;
public?: boolean;
saltUserIds?: boolean;
+ blockBots?: boolean;
};
}>,
reply: FastifyReply
@@ -23,6 +24,7 @@ export async function addSite(
organizationId,
public: isPublic,
saltUserIds,
+ blockBots,
} = request.body;
// Validate domain format using regex
@@ -97,6 +99,7 @@ export async function addSite(
organizationId,
public: isPublic || false,
saltUserIds: saltUserIds || false,
+ blockBots: blockBots === undefined ? true : blockBots,
})
.returning();
@@ -108,6 +111,8 @@ export async function addSite(
public: newSite[0].public || false,
saltUserIds: newSite[0].saltUserIds || false,
domain: newSite[0].domain,
+ blockBots:
+ newSite[0].blockBots === undefined ? true : newSite[0].blockBots,
});
return reply.status(201).send(newSite[0]);
diff --git a/server/src/api/sites/changeSiteBlockBots.ts b/server/src/api/sites/changeSiteBlockBots.ts
new file mode 100644
index 000000000..6f4de7761
--- /dev/null
+++ b/server/src/api/sites/changeSiteBlockBots.ts
@@ -0,0 +1,56 @@
+import { eq } from "drizzle-orm";
+import { FastifyReply, FastifyRequest } from "fastify";
+import { db } from "../../db/postgres/postgres.js";
+import { sites } from "../../db/postgres/schema.js";
+import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
+import { siteConfig } from "../../lib/siteConfig.js";
+
+interface ChangeSiteBlockBotsRequest {
+ Body: {
+ siteId: number;
+ blockBots: boolean;
+ };
+}
+
+export async function changeSiteBlockBots(
+ request: FastifyRequest<ChangeSiteBlockBotsRequest>,
+ reply: FastifyReply
+) {
+ const { siteId, blockBots } = request.body;
+
+ const userHasAdminAccessToSite = await getUserHasAdminAccessToSite(
+ request,
+ String(siteId)
+ );
+ if (!userHasAdminAccessToSite) {
+ return reply.status(403).send({ error: "Forbidden" });
+ }
+
+ try {
+ // Fetch site to check it exists
+ const site = await db.query.sites.findFirst({
+ where: eq(sites.siteId, siteId),
+ });
+
+ if (!site) {
+ return reply.status(404).send({ error: "Site not found" });
+ }
+
+ // Update site blockBots setting
+ await db
+ .update(sites)
+ .set({
+ blockBots: blockBots,
+ updatedAt: new Date(),
+ })
+ .where(eq(sites.siteId, siteId));
+
+ // Update the site config cache
+ siteConfig.updateSiteBlockBotsSetting(siteId, blockBots);
+
+ return reply.status(200).send({ success: true });
+ } catch (error) {
+ console.error("Error changing site bot blocking setting:", error);
+ return reply.status(500).send({ error: "Internal server error" });
+ }
+}
diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.ts
index 5c570bbea..e052eb6f0 100644
--- a/server/src/db/postgres/schema.ts
+++ b/server/src/db/postgres/schema.ts
@@ -63,6 +63,7 @@ export const sites = pgTable(
organizationId: text("organization_id").references(() => organization.id),
public: boolean().default(false),
saltUserIds: boolean().default(false),
+ blockBots: boolean().default(true).notNull(),
},
(table) => [
foreignKey({
diff --git a/server/src/index.ts b/server/src/index.ts
index e65de4c78..3d8cc5d89 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -34,6 +34,7 @@ import { addSite } from "./api/sites/addSite.js";
import { changeSiteDomain } from "./api/sites/changeSiteDomain.js";
import { changeSitePublic } from "./api/sites/changeSitePublic.js";
import { changeSiteSalt } from "./api/sites/changeSiteSalt.js";
+import { changeSiteBlockBots } from "./api/sites/changeSiteBlockBots.js";
import { deleteSite } from "./api/sites/deleteSite.js";
import { getSite } from "./api/sites/getSite.js";
import { getSiteHasData } from "./api/sites/getSiteHasData.js";
@@ -230,6 +231,7 @@ server.post("/add-site", addSite);
server.post("/change-site-domain", changeSiteDomain);
server.post("/change-site-public", changeSitePublic);
server.post("/change-site-salt", changeSiteSalt);
+server.post("/change-site-block-bots", changeSiteBlockBots);
server.post("/delete-site/:id", deleteSite);
server.get("/get-sites", getSites);
server.get("/get-site/:id", getSite);
diff --git a/server/src/lib/siteConfig.ts b/server/src/lib/siteConfig.ts
index bc1191bb4..5a7b7e031 100644
--- a/server/src/lib/siteConfig.ts
+++ b/server/src/lib/siteConfig.ts
@@ -6,6 +6,7 @@ interface SiteConfigData {
public: boolean;
saltUserIds: boolean;
domain: string;
+ blockBots: boolean;
}
class SiteConfig {
@@ -20,6 +21,7 @@ class SiteConfig {
public: sites.public,
saltUserIds: sites.saltUserIds,
domain: sites.domain,
+ blockBots: sites.blockBots,
})
.from(sites);
@@ -32,6 +34,7 @@ class SiteConfig {
public: site.public || false,
saltUserIds: site.saltUserIds || false,
domain: site.domain || "",
+ blockBots: site.blockBots === undefined ? true : site.blockBots,
});
}
@@ -63,6 +66,16 @@ class SiteConfig {
return config?.saltUserIds || false;
}
+ /**
+ * Check if a site has bot blocking enabled
+ */
+ shouldBlockBots(siteId: string | number): boolean {
+ const numericSiteId = Number(siteId);
+ const config = this.siteConfigMap.get(numericSiteId);
+ // Default to true if configuration is not found (safeguard)
+ return config?.blockBots !== false;
+ }
+
/**
* Get the domain of a site
*/
@@ -80,6 +93,7 @@ class SiteConfig {
public: false,
saltUserIds: false,
domain: "",
+ blockBots: true,
};
config.public = isPublic;
this.siteConfigMap.set(siteId, config);
@@ -93,11 +107,26 @@ class SiteConfig {
public: false,
saltUserIds: false,
domain: "",
+ blockBots: true,
};
config.saltUserIds = saltUserIds;
this.siteConfigMap.set(siteId, config);
}
+ /**
+ * Update the bot blocking setting of a site in the cache
+ */
+ updateSiteBlockBotsSetting(siteId: number, blockBots: boolean): void {
+ const config = this.siteConfigMap.get(siteId) || {
+ public: false,
+ saltUserIds: false,
+ domain: "",
+ blockBots: true,
+ };
+ config.blockBots = blockBots;
+ this.siteConfigMap.set(siteId, config);
+ }
+
/**
* Update the domain of a site in the cache
*/
@@ -106,6 +135,7 @@ class SiteConfig {
public: false,
saltUserIds: false,
domain: "",
+ blockBots: true,
};
config.domain = domain;
this.siteConfigMap.set(siteId, config);
diff --git a/server/src/tracker/trackEvent.ts b/server/src/tracker/trackEvent.ts
index e3c46fcd6..18f75f673 100644
--- a/server/src/tracker/trackEvent.ts
+++ b/server/src/tracker/trackEvent.ts
@@ -1,5 +1,6 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { z, ZodError } from "zod";
+import { isbot } from "isbot";
import {
clearSelfReferrer,
createBasePayload,
@@ -242,6 +243,23 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {
});
}
+ // Make sure the site config is loaded
+ await siteConfig.ensureInitialized();
+
+ // Check if bot blocking is enabled for this site and if the request is from a bot
+ if (siteConfig.shouldBlockBots(validatedPayload.site_id)) {
+ const userAgent = request.headers["user-agent"] as string;
+ if (userAgent && isbot(userAgent)) {
+ console.log(
+ `[Tracking] Bot request filtered for site ${validatedPayload.site_id} from User-Agent: ${userAgent}`
+ );
+ return reply.status(200).send({
+ success: true,
+ message: "Event not tracked - bot detected",
+ });
+ }
+ }
+
// Check if the site has exceeded its monthly limit
if (isSiteOverLimit(validatedPayload.site_id)) {
console.log(